1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Link Layer Control (LLC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 10 * Ursula Braun <ubraun@linux.vnet.ibm.com> 11 */ 12 13 #include <net/tcp.h> 14 #include <rdma/ib_verbs.h> 15 16 #include "smc.h" 17 #include "smc_core.h" 18 #include "smc_clc.h" 19 #include "smc_llc.h" 20 #include "smc_pnet.h" 21 22 #define SMC_LLC_DATA_LEN 40 23 24 struct smc_llc_hdr { 25 struct smc_wr_rx_hdr common; 26 union { 27 struct { 28 u8 length; /* 44 */ 29 #if defined(__BIG_ENDIAN_BITFIELD) 30 u8 reserved:4, 31 add_link_rej_rsn:4; 32 #elif defined(__LITTLE_ENDIAN_BITFIELD) 33 u8 add_link_rej_rsn:4, 34 reserved:4; 35 #endif 36 }; 37 u16 length_v2; /* 44 - 8192*/ 38 }; 39 u8 flags; 40 } __packed; /* format defined in 41 * IBM Shared Memory Communications Version 2 42 * (https://www.ibm.com/support/pages/node/6326337) 43 */ 44 45 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 46 47 struct smc_llc_msg_confirm_link { /* type 0x01 */ 48 struct smc_llc_hdr hd; 49 u8 sender_mac[ETH_ALEN]; 50 u8 sender_gid[SMC_GID_SIZE]; 51 u8 sender_qp_num[3]; 52 u8 link_num; 53 u8 link_uid[SMC_LGR_ID_SIZE]; 54 u8 max_links; 55 u8 reserved[9]; 56 }; 57 58 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 59 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 60 61 #define SMC_LLC_ADD_LNK_MAX_LINKS 2 62 63 struct smc_llc_msg_add_link { /* type 0x02 */ 64 struct smc_llc_hdr hd; 65 u8 sender_mac[ETH_ALEN]; 66 u8 reserved2[2]; 67 u8 sender_gid[SMC_GID_SIZE]; 68 u8 sender_qp_num[3]; 69 u8 link_num; 70 #if defined(__BIG_ENDIAN_BITFIELD) 71 u8 reserved3 : 4, 72 qp_mtu : 4; 73 #elif defined(__LITTLE_ENDIAN_BITFIELD) 74 u8 qp_mtu : 4, 75 reserved3 : 4; 76 #endif 77 u8 initial_psn[3]; 78 u8 reserved[8]; 79 }; 80 81 struct smc_llc_msg_add_link_cont_rt { 82 __be32 rmb_key; 83 __be32 rmb_key_new; 84 __be64 rmb_vaddr_new; 85 }; 86 87 struct smc_llc_msg_add_link_v2_ext { 88 #if defined(__BIG_ENDIAN_BITFIELD) 89 u8 v2_direct : 1, 90 reserved : 7; 91 #elif defined(__LITTLE_ENDIAN_BITFIELD) 92 u8 reserved : 7, 93 v2_direct : 1; 94 #endif 95 u8 reserved2; 96 u8 client_target_gid[SMC_GID_SIZE]; 97 u8 reserved3[8]; 98 u16 num_rkeys; 99 struct smc_llc_msg_add_link_cont_rt rt[]; 100 } __packed; /* format defined in 101 * IBM Shared Memory Communications Version 2 102 * (https://www.ibm.com/support/pages/node/6326337) 103 */ 104 105 struct smc_llc_msg_req_add_link_v2 { 106 struct smc_llc_hdr hd; 107 u8 reserved[20]; 108 u8 gid_cnt; 109 u8 reserved2[3]; 110 u8 gid[][SMC_GID_SIZE]; 111 }; 112 113 #define SMC_LLC_RKEYS_PER_CONT_MSG 2 114 115 struct smc_llc_msg_add_link_cont { /* type 0x03 */ 116 struct smc_llc_hdr hd; 117 u8 link_num; 118 u8 num_rkeys; 119 u8 reserved2[2]; 120 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; 121 u8 reserved[4]; 122 } __packed; /* format defined in RFC7609 */ 123 124 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 125 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 126 127 struct smc_llc_msg_del_link { /* type 0x04 */ 128 struct smc_llc_hdr hd; 129 u8 link_num; 130 __be32 reason; 131 u8 reserved[35]; 132 } __packed; /* format defined in RFC7609 */ 133 134 struct smc_llc_msg_test_link { /* type 0x07 */ 135 struct smc_llc_hdr hd; 136 u8 user_data[16]; 137 u8 reserved[24]; 138 }; 139 140 struct smc_rmb_rtoken { 141 union { 142 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 143 /* is actually the num of rtokens, first */ 144 /* rtoken is always for the current link */ 145 u8 link_id; /* link id of the rtoken */ 146 }; 147 __be32 rmb_key; 148 __be64 rmb_vaddr; 149 } __packed; /* format defined in RFC7609 */ 150 151 #define SMC_LLC_RKEYS_PER_MSG 3 152 #define SMC_LLC_RKEYS_PER_MSG_V2 255 153 154 struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 155 struct smc_llc_hdr hd; 156 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 157 u8 reserved; 158 }; 159 160 #define SMC_LLC_DEL_RKEY_MAX 8 161 #define SMC_LLC_FLAG_RKEY_RETRY 0x10 162 #define SMC_LLC_FLAG_RKEY_NEG 0x20 163 164 struct smc_llc_msg_delete_rkey { /* type 0x09 */ 165 struct smc_llc_hdr hd; 166 u8 num_rkeys; 167 u8 err_mask; 168 u8 reserved[2]; 169 __be32 rkey[8]; 170 u8 reserved2[4]; 171 }; 172 173 struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */ 174 struct smc_llc_hdr hd; 175 u8 num_rkeys; 176 u8 num_inval_rkeys; 177 u8 reserved[2]; 178 __be32 rkey[]; 179 }; 180 181 union smc_llc_msg { 182 struct smc_llc_msg_confirm_link confirm_link; 183 struct smc_llc_msg_add_link add_link; 184 struct smc_llc_msg_req_add_link_v2 req_add_link; 185 struct smc_llc_msg_add_link_cont add_link_cont; 186 struct smc_llc_msg_del_link delete_link; 187 188 struct smc_llc_msg_confirm_rkey confirm_rkey; 189 struct smc_llc_msg_delete_rkey delete_rkey; 190 191 struct smc_llc_msg_test_link test_link; 192 struct { 193 struct smc_llc_hdr hdr; 194 u8 data[SMC_LLC_DATA_LEN]; 195 } raw; 196 }; 197 198 #define SMC_LLC_FLAG_RESP 0x80 199 200 struct smc_llc_qentry { 201 struct list_head list; 202 struct smc_link *link; 203 union smc_llc_msg msg; 204 }; 205 206 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); 207 208 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 209 { 210 struct smc_llc_qentry *qentry = flow->qentry; 211 212 flow->qentry = NULL; 213 return qentry; 214 } 215 216 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 217 { 218 struct smc_llc_qentry *qentry; 219 220 if (flow->qentry) { 221 qentry = flow->qentry; 222 flow->qentry = NULL; 223 kfree(qentry); 224 } 225 } 226 227 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 228 struct smc_llc_qentry *qentry) 229 { 230 flow->qentry = qentry; 231 } 232 233 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, 234 struct smc_llc_qentry *qentry) 235 { 236 u8 msg_type = qentry->msg.raw.hdr.common.llc_type; 237 238 if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) && 239 flow_type != msg_type && !lgr->delayed_event) { 240 lgr->delayed_event = qentry; 241 return; 242 } 243 /* drop parallel or already-in-progress llc requests */ 244 if (flow_type != msg_type) 245 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel " 246 "LLC msg: msg %d flow %d role %d\n", 247 SMC_LGR_ID_SIZE, &lgr->id, 248 lgr->net->net_cookie, 249 qentry->msg.raw.hdr.common.type, 250 flow_type, lgr->role); 251 kfree(qentry); 252 } 253 254 /* try to start a new llc flow, initiated by an incoming llc msg */ 255 static bool smc_llc_flow_start(struct smc_llc_flow *flow, 256 struct smc_llc_qentry *qentry) 257 { 258 struct smc_link_group *lgr = qentry->link->lgr; 259 260 spin_lock_bh(&lgr->llc_flow_lock); 261 if (flow->type) { 262 /* a flow is already active */ 263 smc_llc_flow_parallel(lgr, flow->type, qentry); 264 spin_unlock_bh(&lgr->llc_flow_lock); 265 return false; 266 } 267 switch (qentry->msg.raw.hdr.common.llc_type) { 268 case SMC_LLC_ADD_LINK: 269 flow->type = SMC_LLC_FLOW_ADD_LINK; 270 break; 271 case SMC_LLC_DELETE_LINK: 272 flow->type = SMC_LLC_FLOW_DEL_LINK; 273 break; 274 case SMC_LLC_CONFIRM_RKEY: 275 case SMC_LLC_DELETE_RKEY: 276 flow->type = SMC_LLC_FLOW_RKEY; 277 break; 278 default: 279 flow->type = SMC_LLC_FLOW_NONE; 280 } 281 smc_llc_flow_qentry_set(flow, qentry); 282 spin_unlock_bh(&lgr->llc_flow_lock); 283 return true; 284 } 285 286 /* start a new local llc flow, wait till current flow finished */ 287 int smc_llc_flow_initiate(struct smc_link_group *lgr, 288 enum smc_llc_flowtype type) 289 { 290 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 291 int rc; 292 293 /* all flows except confirm_rkey and delete_rkey are exclusive, 294 * confirm/delete rkey flows can run concurrently (local and remote) 295 */ 296 if (type == SMC_LLC_FLOW_RKEY) 297 allowed_remote = SMC_LLC_FLOW_RKEY; 298 again: 299 if (list_empty(&lgr->list)) 300 return -ENODEV; 301 spin_lock_bh(&lgr->llc_flow_lock); 302 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 303 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 304 lgr->llc_flow_rmt.type == allowed_remote)) { 305 lgr->llc_flow_lcl.type = type; 306 spin_unlock_bh(&lgr->llc_flow_lock); 307 return 0; 308 } 309 spin_unlock_bh(&lgr->llc_flow_lock); 310 rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) || 311 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 312 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 313 lgr->llc_flow_rmt.type == allowed_remote))), 314 SMC_LLC_WAIT_TIME * 10); 315 if (!rc) 316 return -ETIMEDOUT; 317 goto again; 318 } 319 320 /* finish the current llc flow */ 321 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 322 { 323 spin_lock_bh(&lgr->llc_flow_lock); 324 memset(flow, 0, sizeof(*flow)); 325 flow->type = SMC_LLC_FLOW_NONE; 326 spin_unlock_bh(&lgr->llc_flow_lock); 327 if (!list_empty(&lgr->list) && lgr->delayed_event && 328 flow == &lgr->llc_flow_lcl) 329 schedule_work(&lgr->llc_event_work); 330 else 331 wake_up(&lgr->llc_flow_waiter); 332 } 333 334 /* lnk is optional and used for early wakeup when link goes down, useful in 335 * cases where we wait for a response on the link after we sent a request 336 */ 337 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 338 struct smc_link *lnk, 339 int time_out, u8 exp_msg) 340 { 341 struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 342 u8 rcv_msg; 343 344 wait_event_timeout(lgr->llc_msg_waiter, 345 (flow->qentry || 346 (lnk && !smc_link_usable(lnk)) || 347 list_empty(&lgr->list)), 348 time_out); 349 if (!flow->qentry || 350 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 351 smc_llc_flow_qentry_del(flow); 352 goto out; 353 } 354 rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type; 355 if (exp_msg && rcv_msg != exp_msg) { 356 if (exp_msg == SMC_LLC_ADD_LINK && 357 rcv_msg == SMC_LLC_DELETE_LINK) { 358 /* flow_start will delay the unexpected msg */ 359 smc_llc_flow_start(&lgr->llc_flow_lcl, 360 smc_llc_flow_qentry_clr(flow)); 361 return NULL; 362 } 363 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: " 364 "msg %d exp %d flow %d role %d flags %x\n", 365 SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie, 366 rcv_msg, exp_msg, 367 flow->type, lgr->role, 368 flow->qentry->msg.raw.hdr.flags); 369 smc_llc_flow_qentry_del(flow); 370 } 371 out: 372 return flow->qentry; 373 } 374 375 /********************************** send *************************************/ 376 377 struct smc_llc_tx_pend { 378 }; 379 380 /* handler for send/transmission completion of an LLC msg */ 381 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 382 struct smc_link *link, 383 enum ib_wc_status wc_status) 384 { 385 /* future work: handle wc_status error for recovery and failover */ 386 } 387 388 /** 389 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 390 * @link: Pointer to SMC link used for sending LLC control message. 391 * @wr_buf: Out variable returning pointer to work request payload buffer. 392 * @pend: Out variable returning pointer to private pending WR tracking. 393 * It's the context the transmit complete handler will get. 394 * 395 * Reserves and pre-fills an entry for a pending work request send/tx. 396 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 397 * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 398 * 399 * Return: 0 on success, otherwise an error value. 400 */ 401 static int smc_llc_add_pending_send(struct smc_link *link, 402 struct smc_wr_buf **wr_buf, 403 struct smc_wr_tx_pend_priv **pend) 404 { 405 int rc; 406 407 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 408 pend); 409 if (rc < 0) 410 return rc; 411 BUILD_BUG_ON_MSG( 412 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 413 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 414 BUILD_BUG_ON_MSG( 415 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 416 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 417 BUILD_BUG_ON_MSG( 418 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 419 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 420 return 0; 421 } 422 423 static int smc_llc_add_pending_send_v2(struct smc_link *link, 424 struct smc_wr_v2_buf **wr_buf, 425 struct smc_wr_tx_pend_priv **pend) 426 { 427 int rc; 428 429 rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend); 430 if (rc < 0) 431 return rc; 432 return 0; 433 } 434 435 static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr, 436 struct smc_link_group *lgr, size_t len) 437 { 438 if (lgr->smc_version == SMC_V2) { 439 hdr->common.llc_version = SMC_V2; 440 hdr->length_v2 = len; 441 } else { 442 hdr->common.llc_version = 0; 443 hdr->length = len; 444 } 445 } 446 447 /* high-level API to send LLC confirm link */ 448 int smc_llc_send_confirm_link(struct smc_link *link, 449 enum smc_llc_reqresp reqresp) 450 { 451 struct smc_llc_msg_confirm_link *confllc; 452 struct smc_wr_tx_pend_priv *pend; 453 struct smc_wr_buf *wr_buf; 454 int rc; 455 456 if (!smc_wr_tx_link_hold(link)) 457 return -ENOLINK; 458 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 459 if (rc) 460 goto put_out; 461 confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 462 memset(confllc, 0, sizeof(*confllc)); 463 confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK; 464 smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc)); 465 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 466 if (reqresp == SMC_LLC_RESP) 467 confllc->hd.flags |= SMC_LLC_FLAG_RESP; 468 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 469 ETH_ALEN); 470 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 471 hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 472 confllc->link_num = link->link_id; 473 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); 474 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; 475 /* send llc message */ 476 rc = smc_wr_tx_send(link, pend); 477 put_out: 478 smc_wr_tx_link_put(link); 479 return rc; 480 } 481 482 /* send LLC confirm rkey request */ 483 static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 484 struct smc_buf_desc *rmb_desc) 485 { 486 struct smc_llc_msg_confirm_rkey *rkeyllc; 487 struct smc_wr_tx_pend_priv *pend; 488 struct smc_wr_buf *wr_buf; 489 struct smc_link *link; 490 int i, rc, rtok_ix; 491 492 if (!smc_wr_tx_link_hold(send_link)) 493 return -ENOLINK; 494 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 495 if (rc) 496 goto put_out; 497 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 498 memset(rkeyllc, 0, sizeof(*rkeyllc)); 499 rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY; 500 smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc)); 501 502 rtok_ix = 1; 503 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 504 link = &send_link->lgr->lnk[i]; 505 if (smc_link_active(link) && link != send_link) { 506 rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 507 rkeyllc->rtoken[rtok_ix].rmb_key = 508 htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 509 rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 510 (u64)sg_dma_address( 511 rmb_desc->sgt[link->link_idx].sgl)); 512 rtok_ix++; 513 } 514 } 515 /* rkey of send_link is in rtoken[0] */ 516 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 517 rkeyllc->rtoken[0].rmb_key = 518 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 519 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 520 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 521 /* send llc message */ 522 rc = smc_wr_tx_send(send_link, pend); 523 put_out: 524 smc_wr_tx_link_put(send_link); 525 return rc; 526 } 527 528 /* send LLC delete rkey request */ 529 static int smc_llc_send_delete_rkey(struct smc_link *link, 530 struct smc_buf_desc *rmb_desc) 531 { 532 struct smc_llc_msg_delete_rkey *rkeyllc; 533 struct smc_wr_tx_pend_priv *pend; 534 struct smc_wr_buf *wr_buf; 535 int rc; 536 537 if (!smc_wr_tx_link_hold(link)) 538 return -ENOLINK; 539 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 540 if (rc) 541 goto put_out; 542 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 543 memset(rkeyllc, 0, sizeof(*rkeyllc)); 544 rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY; 545 smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc)); 546 rkeyllc->num_rkeys = 1; 547 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 548 /* send llc message */ 549 rc = smc_wr_tx_send(link, pend); 550 put_out: 551 smc_wr_tx_link_put(link); 552 return rc; 553 } 554 555 /* return first buffer from any of the next buf lists */ 556 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, 557 int *buf_lst) 558 { 559 struct smc_buf_desc *buf_pos; 560 561 while (*buf_lst < SMC_RMBE_SIZES) { 562 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], 563 struct smc_buf_desc, list); 564 if (buf_pos) 565 return buf_pos; 566 (*buf_lst)++; 567 } 568 return NULL; 569 } 570 571 /* return next rmb from buffer lists */ 572 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, 573 int *buf_lst, 574 struct smc_buf_desc *buf_pos) 575 { 576 struct smc_buf_desc *buf_next; 577 578 if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { 579 (*buf_lst)++; 580 return _smc_llc_get_next_rmb(lgr, buf_lst); 581 } 582 buf_next = list_next_entry(buf_pos, list); 583 return buf_next; 584 } 585 586 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, 587 int *buf_lst) 588 { 589 *buf_lst = 0; 590 return smc_llc_get_next_rmb(lgr, buf_lst, NULL); 591 } 592 593 static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext, 594 struct smc_link *link, struct smc_link *link_new) 595 { 596 struct smc_link_group *lgr = link->lgr; 597 struct smc_buf_desc *buf_pos; 598 int prim_lnk_idx, lnk_idx, i; 599 struct smc_buf_desc *rmb; 600 int len = sizeof(*ext); 601 int buf_lst; 602 603 ext->v2_direct = !lgr->uses_gateway; 604 memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE); 605 606 prim_lnk_idx = link->link_idx; 607 lnk_idx = link_new->link_idx; 608 mutex_lock(&lgr->rmbs_lock); 609 ext->num_rkeys = lgr->conns_num; 610 if (!ext->num_rkeys) 611 goto out; 612 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 613 for (i = 0; i < ext->num_rkeys; i++) { 614 if (!buf_pos) 615 break; 616 rmb = buf_pos; 617 ext->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); 618 ext->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); 619 ext->rt[i].rmb_vaddr_new = 620 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 621 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos); 622 while (buf_pos && !(buf_pos)->used) 623 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos); 624 } 625 len += i * sizeof(ext->rt[0]); 626 out: 627 mutex_unlock(&lgr->rmbs_lock); 628 return len; 629 } 630 631 /* send ADD LINK request or response */ 632 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 633 struct smc_link *link_new, 634 enum smc_llc_reqresp reqresp) 635 { 636 struct smc_llc_msg_add_link_v2_ext *ext = NULL; 637 struct smc_llc_msg_add_link *addllc; 638 struct smc_wr_tx_pend_priv *pend; 639 int len = sizeof(*addllc); 640 int rc; 641 642 if (!smc_wr_tx_link_hold(link)) 643 return -ENOLINK; 644 if (link->lgr->smc_version == SMC_V2) { 645 struct smc_wr_v2_buf *wr_buf; 646 647 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend); 648 if (rc) 649 goto put_out; 650 addllc = (struct smc_llc_msg_add_link *)wr_buf; 651 ext = (struct smc_llc_msg_add_link_v2_ext *) 652 &wr_buf->raw[sizeof(*addllc)]; 653 memset(ext, 0, SMC_WR_TX_SIZE); 654 } else { 655 struct smc_wr_buf *wr_buf; 656 657 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 658 if (rc) 659 goto put_out; 660 addllc = (struct smc_llc_msg_add_link *)wr_buf; 661 } 662 663 memset(addllc, 0, sizeof(*addllc)); 664 addllc->hd.common.llc_type = SMC_LLC_ADD_LINK; 665 if (reqresp == SMC_LLC_RESP) 666 addllc->hd.flags |= SMC_LLC_FLAG_RESP; 667 memcpy(addllc->sender_mac, mac, ETH_ALEN); 668 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 669 if (link_new) { 670 addllc->link_num = link_new->link_id; 671 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 672 hton24(addllc->initial_psn, link_new->psn_initial); 673 if (reqresp == SMC_LLC_REQ) 674 addllc->qp_mtu = link_new->path_mtu; 675 else 676 addllc->qp_mtu = min(link_new->path_mtu, 677 link_new->peer_mtu); 678 } 679 if (ext && link_new) 680 len += smc_llc_fill_ext_v2(ext, link, link_new); 681 smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len); 682 /* send llc message */ 683 if (link->lgr->smc_version == SMC_V2) 684 rc = smc_wr_tx_v2_send(link, pend, len); 685 else 686 rc = smc_wr_tx_send(link, pend); 687 put_out: 688 smc_wr_tx_link_put(link); 689 return rc; 690 } 691 692 /* send DELETE LINK request or response */ 693 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 694 enum smc_llc_reqresp reqresp, bool orderly, 695 u32 reason) 696 { 697 struct smc_llc_msg_del_link *delllc; 698 struct smc_wr_tx_pend_priv *pend; 699 struct smc_wr_buf *wr_buf; 700 int rc; 701 702 if (!smc_wr_tx_link_hold(link)) 703 return -ENOLINK; 704 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 705 if (rc) 706 goto put_out; 707 delllc = (struct smc_llc_msg_del_link *)wr_buf; 708 709 memset(delllc, 0, sizeof(*delllc)); 710 delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK; 711 smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc)); 712 if (reqresp == SMC_LLC_RESP) 713 delllc->hd.flags |= SMC_LLC_FLAG_RESP; 714 if (orderly) 715 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 716 if (link_del_id) 717 delllc->link_num = link_del_id; 718 else 719 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 720 delllc->reason = htonl(reason); 721 /* send llc message */ 722 rc = smc_wr_tx_send(link, pend); 723 put_out: 724 smc_wr_tx_link_put(link); 725 return rc; 726 } 727 728 /* send LLC test link request */ 729 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 730 { 731 struct smc_llc_msg_test_link *testllc; 732 struct smc_wr_tx_pend_priv *pend; 733 struct smc_wr_buf *wr_buf; 734 int rc; 735 736 if (!smc_wr_tx_link_hold(link)) 737 return -ENOLINK; 738 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 739 if (rc) 740 goto put_out; 741 testllc = (struct smc_llc_msg_test_link *)wr_buf; 742 memset(testllc, 0, sizeof(*testllc)); 743 testllc->hd.common.llc_type = SMC_LLC_TEST_LINK; 744 smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc)); 745 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 746 /* send llc message */ 747 rc = smc_wr_tx_send(link, pend); 748 put_out: 749 smc_wr_tx_link_put(link); 750 return rc; 751 } 752 753 /* schedule an llc send on link, may wait for buffers */ 754 static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 755 { 756 struct smc_wr_tx_pend_priv *pend; 757 struct smc_wr_buf *wr_buf; 758 int rc; 759 760 if (!smc_wr_tx_link_hold(link)) 761 return -ENOLINK; 762 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 763 if (rc) 764 goto put_out; 765 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 766 rc = smc_wr_tx_send(link, pend); 767 put_out: 768 smc_wr_tx_link_put(link); 769 return rc; 770 } 771 772 /* schedule an llc send on link, may wait for buffers, 773 * and wait for send completion notification. 774 * @return 0 on success 775 */ 776 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) 777 { 778 struct smc_wr_tx_pend_priv *pend; 779 struct smc_wr_buf *wr_buf; 780 int rc; 781 782 if (!smc_wr_tx_link_hold(link)) 783 return -ENOLINK; 784 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 785 if (rc) 786 goto put_out; 787 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 788 rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); 789 put_out: 790 smc_wr_tx_link_put(link); 791 return rc; 792 } 793 794 /********************************* receive ***********************************/ 795 796 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 797 enum smc_lgr_type lgr_new_t) 798 { 799 int i; 800 801 if (lgr->type == SMC_LGR_SYMMETRIC || 802 (lgr->type != SMC_LGR_SINGLE && 803 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 804 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 805 return -EMLINK; 806 807 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 808 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 809 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 810 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 811 return i; 812 } else { 813 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 814 if (lgr->lnk[i].state == SMC_LNK_UNUSED) 815 return i; 816 } 817 return -EMLINK; 818 } 819 820 /* send one add_link_continue msg */ 821 static int smc_llc_add_link_cont(struct smc_link *link, 822 struct smc_link *link_new, u8 *num_rkeys_todo, 823 int *buf_lst, struct smc_buf_desc **buf_pos) 824 { 825 struct smc_llc_msg_add_link_cont *addc_llc; 826 struct smc_link_group *lgr = link->lgr; 827 int prim_lnk_idx, lnk_idx, i, rc; 828 struct smc_wr_tx_pend_priv *pend; 829 struct smc_wr_buf *wr_buf; 830 struct smc_buf_desc *rmb; 831 u8 n; 832 833 if (!smc_wr_tx_link_hold(link)) 834 return -ENOLINK; 835 rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 836 if (rc) 837 goto put_out; 838 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; 839 memset(addc_llc, 0, sizeof(*addc_llc)); 840 841 prim_lnk_idx = link->link_idx; 842 lnk_idx = link_new->link_idx; 843 addc_llc->link_num = link_new->link_id; 844 addc_llc->num_rkeys = *num_rkeys_todo; 845 n = *num_rkeys_todo; 846 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { 847 if (!*buf_pos) { 848 addc_llc->num_rkeys = addc_llc->num_rkeys - 849 *num_rkeys_todo; 850 *num_rkeys_todo = 0; 851 break; 852 } 853 rmb = *buf_pos; 854 855 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); 856 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); 857 addc_llc->rt[i].rmb_vaddr_new = 858 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 859 860 (*num_rkeys_todo)--; 861 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 862 while (*buf_pos && !(*buf_pos)->used) 863 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 864 } 865 addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT; 866 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); 867 if (lgr->role == SMC_CLNT) 868 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; 869 rc = smc_wr_tx_send(link, pend); 870 put_out: 871 smc_wr_tx_link_put(link); 872 return rc; 873 } 874 875 static int smc_llc_cli_rkey_exchange(struct smc_link *link, 876 struct smc_link *link_new) 877 { 878 struct smc_llc_msg_add_link_cont *addc_llc; 879 struct smc_link_group *lgr = link->lgr; 880 u8 max, num_rkeys_send, num_rkeys_recv; 881 struct smc_llc_qentry *qentry; 882 struct smc_buf_desc *buf_pos; 883 int buf_lst; 884 int rc = 0; 885 int i; 886 887 mutex_lock(&lgr->rmbs_lock); 888 num_rkeys_send = lgr->conns_num; 889 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 890 do { 891 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, 892 SMC_LLC_ADD_LINK_CONT); 893 if (!qentry) { 894 rc = -ETIMEDOUT; 895 break; 896 } 897 addc_llc = &qentry->msg.add_link_cont; 898 num_rkeys_recv = addc_llc->num_rkeys; 899 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 900 for (i = 0; i < max; i++) { 901 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 902 addc_llc->rt[i].rmb_key, 903 addc_llc->rt[i].rmb_vaddr_new, 904 addc_llc->rt[i].rmb_key_new); 905 num_rkeys_recv--; 906 } 907 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 908 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 909 &buf_lst, &buf_pos); 910 if (rc) 911 break; 912 } while (num_rkeys_send || num_rkeys_recv); 913 914 mutex_unlock(&lgr->rmbs_lock); 915 return rc; 916 } 917 918 /* prepare and send an add link reject response */ 919 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) 920 { 921 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 922 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 923 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 924 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr, 925 sizeof(qentry->msg)); 926 return smc_llc_send_message(qentry->link, &qentry->msg); 927 } 928 929 static int smc_llc_cli_conf_link(struct smc_link *link, 930 struct smc_init_info *ini, 931 struct smc_link *link_new, 932 enum smc_lgr_type lgr_new_t) 933 { 934 struct smc_link_group *lgr = link->lgr; 935 struct smc_llc_qentry *qentry = NULL; 936 int rc = 0; 937 938 /* receive CONFIRM LINK request over RoCE fabric */ 939 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); 940 if (!qentry) { 941 rc = smc_llc_send_delete_link(link, link_new->link_id, 942 SMC_LLC_REQ, false, 943 SMC_LLC_DEL_LOST_PATH); 944 return -ENOLINK; 945 } 946 if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) { 947 /* received DELETE_LINK instead */ 948 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 949 smc_llc_send_message(link, &qentry->msg); 950 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 951 return -ENOLINK; 952 } 953 smc_llc_save_peer_uid(qentry); 954 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 955 956 rc = smc_ib_modify_qp_rts(link_new); 957 if (rc) { 958 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 959 false, SMC_LLC_DEL_LOST_PATH); 960 return -ENOLINK; 961 } 962 smc_wr_remember_qp_attr(link_new); 963 964 rc = smcr_buf_reg_lgr(link_new); 965 if (rc) { 966 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 967 false, SMC_LLC_DEL_LOST_PATH); 968 return -ENOLINK; 969 } 970 971 /* send CONFIRM LINK response over RoCE fabric */ 972 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); 973 if (rc) { 974 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 975 false, SMC_LLC_DEL_LOST_PATH); 976 return -ENOLINK; 977 } 978 smc_llc_link_active(link_new); 979 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 980 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 981 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 982 else 983 smcr_lgr_set_type(lgr, lgr_new_t); 984 return 0; 985 } 986 987 static void smc_llc_save_add_link_rkeys(struct smc_link *link, 988 struct smc_link *link_new) 989 { 990 struct smc_llc_msg_add_link_v2_ext *ext; 991 struct smc_link_group *lgr = link->lgr; 992 int max, i; 993 994 ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 + 995 SMC_WR_TX_SIZE); 996 max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); 997 mutex_lock(&lgr->rmbs_lock); 998 for (i = 0; i < max; i++) { 999 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 1000 ext->rt[i].rmb_key, 1001 ext->rt[i].rmb_vaddr_new, 1002 ext->rt[i].rmb_key_new); 1003 } 1004 mutex_unlock(&lgr->rmbs_lock); 1005 } 1006 1007 static void smc_llc_save_add_link_info(struct smc_link *link, 1008 struct smc_llc_msg_add_link *add_llc) 1009 { 1010 link->peer_qpn = ntoh24(add_llc->sender_qp_num); 1011 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); 1012 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); 1013 link->peer_psn = ntoh24(add_llc->initial_psn); 1014 link->peer_mtu = add_llc->qp_mtu; 1015 } 1016 1017 /* as an SMC client, process an add link request */ 1018 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) 1019 { 1020 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; 1021 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 1022 struct smc_link_group *lgr = smc_get_lgr(link); 1023 struct smc_init_info *ini = NULL; 1024 struct smc_link *lnk_new = NULL; 1025 int lnk_idx, rc = 0; 1026 1027 if (!llc->qp_mtu) 1028 goto out_reject; 1029 1030 ini = kzalloc(sizeof(*ini), GFP_KERNEL); 1031 if (!ini) { 1032 rc = -ENOMEM; 1033 goto out_reject; 1034 } 1035 1036 ini->vlan_id = lgr->vlan_id; 1037 if (lgr->smc_version == SMC_V2) { 1038 ini->check_smcrv2 = true; 1039 ini->smcrv2.saddr = lgr->saddr; 1040 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid); 1041 } 1042 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1043 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1044 (lgr->smc_version == SMC_V2 || 1045 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) { 1046 if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2) 1047 goto out_reject; 1048 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 1049 } 1050 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) { 1051 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1052 ini->smcrv2.ib_dev_v2 = link->smcibdev; 1053 ini->smcrv2.ib_port_v2 = link->ibport; 1054 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) { 1055 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1056 ini->ib_dev = link->smcibdev; 1057 ini->ib_port = link->ibport; 1058 } 1059 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1060 if (lnk_idx < 0) 1061 goto out_reject; 1062 lnk_new = &lgr->lnk[lnk_idx]; 1063 rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini); 1064 if (rc) 1065 goto out_reject; 1066 smc_llc_save_add_link_info(lnk_new, llc); 1067 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ 1068 smc_llc_link_set_uid(lnk_new); 1069 1070 rc = smc_ib_ready_link(lnk_new); 1071 if (rc) 1072 goto out_clear_lnk; 1073 1074 rc = smcr_buf_map_lgr(lnk_new); 1075 if (rc) 1076 goto out_clear_lnk; 1077 1078 rc = smc_llc_send_add_link(link, 1079 lnk_new->smcibdev->mac[lnk_new->ibport - 1], 1080 lnk_new->gid, lnk_new, SMC_LLC_RESP); 1081 if (rc) 1082 goto out_clear_lnk; 1083 if (lgr->smc_version == SMC_V2) { 1084 smc_llc_save_add_link_rkeys(link, lnk_new); 1085 } else { 1086 rc = smc_llc_cli_rkey_exchange(link, lnk_new); 1087 if (rc) { 1088 rc = 0; 1089 goto out_clear_lnk; 1090 } 1091 } 1092 rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t); 1093 if (!rc) 1094 goto out; 1095 out_clear_lnk: 1096 lnk_new->state = SMC_LNK_INACTIVE; 1097 smcr_link_clear(lnk_new, false); 1098 out_reject: 1099 smc_llc_cli_add_link_reject(qentry); 1100 out: 1101 kfree(ini); 1102 kfree(qentry); 1103 return rc; 1104 } 1105 1106 static void smc_llc_send_request_add_link(struct smc_link *link) 1107 { 1108 struct smc_llc_msg_req_add_link_v2 *llc; 1109 struct smc_wr_tx_pend_priv *pend; 1110 struct smc_wr_v2_buf *wr_buf; 1111 struct smc_gidlist gidlist; 1112 int rc, len, i; 1113 1114 if (!smc_wr_tx_link_hold(link)) 1115 return; 1116 if (link->lgr->type == SMC_LGR_SYMMETRIC || 1117 link->lgr->type == SMC_LGR_ASYMMETRIC_PEER) 1118 goto put_out; 1119 1120 smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid); 1121 if (gidlist.len <= 1) 1122 goto put_out; 1123 1124 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend); 1125 if (rc) 1126 goto put_out; 1127 llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf; 1128 memset(llc, 0, SMC_WR_TX_SIZE); 1129 1130 llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK; 1131 for (i = 0; i < gidlist.len; i++) 1132 memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0])); 1133 llc->gid_cnt = gidlist.len; 1134 len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0])); 1135 smc_llc_init_msg_hdr(&llc->hd, link->lgr, len); 1136 rc = smc_wr_tx_v2_send(link, pend, len); 1137 if (!rc) 1138 /* set REQ_ADD_LINK flow and wait for response from peer */ 1139 link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK; 1140 put_out: 1141 smc_wr_tx_link_put(link); 1142 } 1143 1144 /* as an SMC client, invite server to start the add_link processing */ 1145 static void smc_llc_cli_add_link_invite(struct smc_link *link, 1146 struct smc_llc_qentry *qentry) 1147 { 1148 struct smc_link_group *lgr = smc_get_lgr(link); 1149 struct smc_init_info *ini = NULL; 1150 1151 if (lgr->smc_version == SMC_V2) { 1152 smc_llc_send_request_add_link(link); 1153 goto out; 1154 } 1155 1156 if (lgr->type == SMC_LGR_SYMMETRIC || 1157 lgr->type == SMC_LGR_ASYMMETRIC_PEER) 1158 goto out; 1159 1160 ini = kzalloc(sizeof(*ini), GFP_KERNEL); 1161 if (!ini) 1162 goto out; 1163 1164 ini->vlan_id = lgr->vlan_id; 1165 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1166 if (!ini->ib_dev) 1167 goto out; 1168 1169 smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1], 1170 ini->ib_gid, NULL, SMC_LLC_REQ); 1171 out: 1172 kfree(ini); 1173 kfree(qentry); 1174 } 1175 1176 static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc) 1177 { 1178 int i; 1179 1180 for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++) 1181 if (llc->raw.data[i]) 1182 return false; 1183 return true; 1184 } 1185 1186 static bool smc_llc_is_local_add_link(union smc_llc_msg *llc) 1187 { 1188 if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK && 1189 smc_llc_is_empty_llc_message(llc)) 1190 return true; 1191 return false; 1192 } 1193 1194 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) 1195 { 1196 struct smc_llc_qentry *qentry; 1197 1198 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1199 1200 mutex_lock(&lgr->llc_conf_mutex); 1201 if (smc_llc_is_local_add_link(&qentry->msg)) 1202 smc_llc_cli_add_link_invite(qentry->link, qentry); 1203 else 1204 smc_llc_cli_add_link(qentry->link, qentry); 1205 mutex_unlock(&lgr->llc_conf_mutex); 1206 } 1207 1208 static int smc_llc_active_link_count(struct smc_link_group *lgr) 1209 { 1210 int i, link_count = 0; 1211 1212 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1213 if (!smc_link_active(&lgr->lnk[i])) 1214 continue; 1215 link_count++; 1216 } 1217 return link_count; 1218 } 1219 1220 /* find the asymmetric link when 3 links are established */ 1221 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) 1222 { 1223 int asym_idx = -ENOENT; 1224 int i, j, k; 1225 bool found; 1226 1227 /* determine asymmetric link */ 1228 found = false; 1229 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1230 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { 1231 if (!smc_link_usable(&lgr->lnk[i]) || 1232 !smc_link_usable(&lgr->lnk[j])) 1233 continue; 1234 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, 1235 SMC_GID_SIZE)) { 1236 found = true; /* asym_lnk is i or j */ 1237 break; 1238 } 1239 } 1240 if (found) 1241 break; 1242 } 1243 if (!found) 1244 goto out; /* no asymmetric link */ 1245 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { 1246 if (!smc_link_usable(&lgr->lnk[k])) 1247 continue; 1248 if (k != i && 1249 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, 1250 SMC_GID_SIZE)) { 1251 asym_idx = i; 1252 break; 1253 } 1254 if (k != j && 1255 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, 1256 SMC_GID_SIZE)) { 1257 asym_idx = j; 1258 break; 1259 } 1260 } 1261 out: 1262 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; 1263 } 1264 1265 static void smc_llc_delete_asym_link(struct smc_link_group *lgr) 1266 { 1267 struct smc_link *lnk_new = NULL, *lnk_asym; 1268 struct smc_llc_qentry *qentry; 1269 int rc; 1270 1271 lnk_asym = smc_llc_find_asym_link(lgr); 1272 if (!lnk_asym) 1273 return; /* no asymmetric link */ 1274 if (!smc_link_downing(&lnk_asym->state)) 1275 return; 1276 lnk_new = smc_switch_conns(lgr, lnk_asym, false); 1277 smc_wr_tx_wait_no_pending_sends(lnk_asym); 1278 if (!lnk_new) 1279 goto out_free; 1280 /* change flow type from ADD_LINK into DEL_LINK */ 1281 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; 1282 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, 1283 true, SMC_LLC_DEL_NO_ASYM_NEEDED); 1284 if (rc) { 1285 smcr_link_down_cond(lnk_new); 1286 goto out_free; 1287 } 1288 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, 1289 SMC_LLC_DELETE_LINK); 1290 if (!qentry) { 1291 smcr_link_down_cond(lnk_new); 1292 goto out_free; 1293 } 1294 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1295 out_free: 1296 smcr_link_clear(lnk_asym, true); 1297 } 1298 1299 static int smc_llc_srv_rkey_exchange(struct smc_link *link, 1300 struct smc_link *link_new) 1301 { 1302 struct smc_llc_msg_add_link_cont *addc_llc; 1303 struct smc_link_group *lgr = link->lgr; 1304 u8 max, num_rkeys_send, num_rkeys_recv; 1305 struct smc_llc_qentry *qentry = NULL; 1306 struct smc_buf_desc *buf_pos; 1307 int buf_lst; 1308 int rc = 0; 1309 int i; 1310 1311 mutex_lock(&lgr->rmbs_lock); 1312 num_rkeys_send = lgr->conns_num; 1313 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 1314 do { 1315 smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 1316 &buf_lst, &buf_pos); 1317 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, 1318 SMC_LLC_ADD_LINK_CONT); 1319 if (!qentry) { 1320 rc = -ETIMEDOUT; 1321 goto out; 1322 } 1323 addc_llc = &qentry->msg.add_link_cont; 1324 num_rkeys_recv = addc_llc->num_rkeys; 1325 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 1326 for (i = 0; i < max; i++) { 1327 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 1328 addc_llc->rt[i].rmb_key, 1329 addc_llc->rt[i].rmb_vaddr_new, 1330 addc_llc->rt[i].rmb_key_new); 1331 num_rkeys_recv--; 1332 } 1333 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1334 } while (num_rkeys_send || num_rkeys_recv); 1335 out: 1336 mutex_unlock(&lgr->rmbs_lock); 1337 return rc; 1338 } 1339 1340 static int smc_llc_srv_conf_link(struct smc_link *link, 1341 struct smc_link *link_new, 1342 enum smc_lgr_type lgr_new_t) 1343 { 1344 struct smc_link_group *lgr = link->lgr; 1345 struct smc_llc_qentry *qentry = NULL; 1346 int rc; 1347 1348 /* send CONFIRM LINK request over the RoCE fabric */ 1349 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); 1350 if (rc) 1351 return -ENOLINK; 1352 /* receive CONFIRM LINK response over the RoCE fabric */ 1353 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0); 1354 if (!qentry || 1355 qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) { 1356 /* send DELETE LINK */ 1357 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 1358 false, SMC_LLC_DEL_LOST_PATH); 1359 if (qentry) 1360 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1361 return -ENOLINK; 1362 } 1363 smc_llc_save_peer_uid(qentry); 1364 smc_llc_link_active(link_new); 1365 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1366 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1367 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1368 else 1369 smcr_lgr_set_type(lgr, lgr_new_t); 1370 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1371 return 0; 1372 } 1373 1374 static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry) 1375 { 1376 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 1377 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr, 1378 sizeof(qentry->msg)); 1379 memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data)); 1380 smc_llc_send_message(qentry->link, &qentry->msg); 1381 } 1382 1383 int smc_llc_srv_add_link(struct smc_link *link, 1384 struct smc_llc_qentry *req_qentry) 1385 { 1386 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 1387 struct smc_link_group *lgr = link->lgr; 1388 struct smc_llc_msg_add_link *add_llc; 1389 struct smc_llc_qentry *qentry = NULL; 1390 bool send_req_add_link_resp = false; 1391 struct smc_link *link_new = NULL; 1392 struct smc_init_info *ini = NULL; 1393 int lnk_idx, rc = 0; 1394 1395 if (req_qentry && 1396 req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK) 1397 send_req_add_link_resp = true; 1398 1399 ini = kzalloc(sizeof(*ini), GFP_KERNEL); 1400 if (!ini) { 1401 rc = -ENOMEM; 1402 goto out; 1403 } 1404 1405 /* ignore client add link recommendation, start new flow */ 1406 ini->vlan_id = lgr->vlan_id; 1407 if (lgr->smc_version == SMC_V2) { 1408 ini->check_smcrv2 = true; 1409 ini->smcrv2.saddr = lgr->saddr; 1410 if (send_req_add_link_resp) { 1411 struct smc_llc_msg_req_add_link_v2 *req_add = 1412 &req_qentry->msg.req_add_link; 1413 1414 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]); 1415 } 1416 } 1417 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1418 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) { 1419 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1420 ini->smcrv2.ib_dev_v2 = link->smcibdev; 1421 ini->smcrv2.ib_port_v2 = link->ibport; 1422 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) { 1423 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1424 ini->ib_dev = link->smcibdev; 1425 ini->ib_port = link->ibport; 1426 } 1427 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1428 if (lnk_idx < 0) { 1429 rc = 0; 1430 goto out; 1431 } 1432 1433 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini); 1434 if (rc) 1435 goto out; 1436 link_new = &lgr->lnk[lnk_idx]; 1437 1438 rc = smcr_buf_map_lgr(link_new); 1439 if (rc) 1440 goto out_err; 1441 1442 rc = smc_llc_send_add_link(link, 1443 link_new->smcibdev->mac[link_new->ibport-1], 1444 link_new->gid, link_new, SMC_LLC_REQ); 1445 if (rc) 1446 goto out_err; 1447 send_req_add_link_resp = false; 1448 /* receive ADD LINK response over the RoCE fabric */ 1449 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); 1450 if (!qentry) { 1451 rc = -ETIMEDOUT; 1452 goto out_err; 1453 } 1454 add_llc = &qentry->msg.add_link; 1455 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { 1456 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1457 rc = -ENOLINK; 1458 goto out_err; 1459 } 1460 if (lgr->type == SMC_LGR_SINGLE && 1461 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1462 (lgr->smc_version == SMC_V2 || 1463 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) { 1464 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 1465 } 1466 smc_llc_save_add_link_info(link_new, add_llc); 1467 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1468 1469 rc = smc_ib_ready_link(link_new); 1470 if (rc) 1471 goto out_err; 1472 rc = smcr_buf_reg_lgr(link_new); 1473 if (rc) 1474 goto out_err; 1475 if (lgr->smc_version == SMC_V2) { 1476 smc_llc_save_add_link_rkeys(link, link_new); 1477 } else { 1478 rc = smc_llc_srv_rkey_exchange(link, link_new); 1479 if (rc) 1480 goto out_err; 1481 } 1482 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); 1483 if (rc) 1484 goto out_err; 1485 kfree(ini); 1486 return 0; 1487 out_err: 1488 if (link_new) { 1489 link_new->state = SMC_LNK_INACTIVE; 1490 smcr_link_clear(link_new, false); 1491 } 1492 out: 1493 kfree(ini); 1494 if (send_req_add_link_resp) 1495 smc_llc_send_req_add_link_response(req_qentry); 1496 return rc; 1497 } 1498 1499 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) 1500 { 1501 struct smc_link *link = lgr->llc_flow_lcl.qentry->link; 1502 struct smc_llc_qentry *qentry; 1503 int rc; 1504 1505 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1506 1507 mutex_lock(&lgr->llc_conf_mutex); 1508 rc = smc_llc_srv_add_link(link, qentry); 1509 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { 1510 /* delete any asymmetric link */ 1511 smc_llc_delete_asym_link(lgr); 1512 } 1513 mutex_unlock(&lgr->llc_conf_mutex); 1514 kfree(qentry); 1515 } 1516 1517 /* enqueue a local add_link req to trigger a new add_link flow */ 1518 void smc_llc_add_link_local(struct smc_link *link) 1519 { 1520 struct smc_llc_msg_add_link add_llc = {}; 1521 1522 add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK; 1523 smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc)); 1524 /* no dev and port needed */ 1525 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); 1526 } 1527 1528 /* worker to process an add link message */ 1529 static void smc_llc_add_link_work(struct work_struct *work) 1530 { 1531 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1532 llc_add_link_work); 1533 1534 if (list_empty(&lgr->list)) { 1535 /* link group is terminating */ 1536 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1537 goto out; 1538 } 1539 1540 if (lgr->role == SMC_CLNT) 1541 smc_llc_process_cli_add_link(lgr); 1542 else 1543 smc_llc_process_srv_add_link(lgr); 1544 out: 1545 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK) 1546 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1547 } 1548 1549 /* enqueue a local del_link msg to trigger a new del_link flow, 1550 * called only for role SMC_SERV 1551 */ 1552 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) 1553 { 1554 struct smc_llc_msg_del_link del_llc = {}; 1555 1556 del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK; 1557 smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc)); 1558 del_llc.link_num = del_link_id; 1559 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); 1560 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1561 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); 1562 } 1563 1564 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) 1565 { 1566 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; 1567 struct smc_llc_msg_del_link *del_llc; 1568 struct smc_llc_qentry *qentry; 1569 int active_links; 1570 int lnk_idx; 1571 1572 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1573 lnk = qentry->link; 1574 del_llc = &qentry->msg.delete_link; 1575 1576 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1577 smc_lgr_terminate_sched(lgr); 1578 goto out; 1579 } 1580 mutex_lock(&lgr->llc_conf_mutex); 1581 /* delete single link */ 1582 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { 1583 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) 1584 continue; 1585 lnk_del = &lgr->lnk[lnk_idx]; 1586 break; 1587 } 1588 del_llc->hd.flags |= SMC_LLC_FLAG_RESP; 1589 if (!lnk_del) { 1590 /* link was not found */ 1591 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); 1592 smc_llc_send_message(lnk, &qentry->msg); 1593 goto out_unlock; 1594 } 1595 lnk_asym = smc_llc_find_asym_link(lgr); 1596 1597 del_llc->reason = 0; 1598 smc_llc_send_message(lnk, &qentry->msg); /* response */ 1599 1600 if (smc_link_downing(&lnk_del->state)) 1601 smc_switch_conns(lgr, lnk_del, false); 1602 smcr_link_clear(lnk_del, true); 1603 1604 active_links = smc_llc_active_link_count(lgr); 1605 if (lnk_del == lnk_asym) { 1606 /* expected deletion of asym link, don't change lgr state */ 1607 } else if (active_links == 1) { 1608 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1609 } else if (!active_links) { 1610 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1611 smc_lgr_terminate_sched(lgr); 1612 } 1613 out_unlock: 1614 mutex_unlock(&lgr->llc_conf_mutex); 1615 out: 1616 kfree(qentry); 1617 } 1618 1619 /* try to send a DELETE LINK ALL request on any active link, 1620 * waiting for send completion 1621 */ 1622 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) 1623 { 1624 struct smc_llc_msg_del_link delllc = {}; 1625 int i; 1626 1627 delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK; 1628 smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc)); 1629 if (ord) 1630 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1631 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 1632 delllc.reason = htonl(rsn); 1633 1634 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1635 if (!smc_link_sendable(&lgr->lnk[i])) 1636 continue; 1637 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) 1638 break; 1639 } 1640 } 1641 1642 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) 1643 { 1644 struct smc_llc_msg_del_link *del_llc; 1645 struct smc_link *lnk, *lnk_del; 1646 struct smc_llc_qentry *qentry; 1647 int active_links; 1648 int i; 1649 1650 mutex_lock(&lgr->llc_conf_mutex); 1651 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1652 lnk = qentry->link; 1653 del_llc = &qentry->msg.delete_link; 1654 1655 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 1656 /* delete entire lgr */ 1657 smc_llc_send_link_delete_all(lgr, true, ntohl( 1658 qentry->msg.delete_link.reason)); 1659 smc_lgr_terminate_sched(lgr); 1660 goto out; 1661 } 1662 /* delete single link */ 1663 lnk_del = NULL; 1664 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1665 if (lgr->lnk[i].link_id == del_llc->link_num) { 1666 lnk_del = &lgr->lnk[i]; 1667 break; 1668 } 1669 } 1670 if (!lnk_del) 1671 goto out; /* asymmetric link already deleted */ 1672 1673 if (smc_link_downing(&lnk_del->state)) { 1674 if (smc_switch_conns(lgr, lnk_del, false)) 1675 smc_wr_tx_wait_no_pending_sends(lnk_del); 1676 } 1677 if (!list_empty(&lgr->list)) { 1678 /* qentry is either a request from peer (send it back to 1679 * initiate the DELETE_LINK processing), or a locally 1680 * enqueued DELETE_LINK request (forward it) 1681 */ 1682 if (!smc_llc_send_message(lnk, &qentry->msg)) { 1683 struct smc_llc_qentry *qentry2; 1684 1685 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, 1686 SMC_LLC_DELETE_LINK); 1687 if (qentry2) 1688 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1689 } 1690 } 1691 smcr_link_clear(lnk_del, true); 1692 1693 active_links = smc_llc_active_link_count(lgr); 1694 if (active_links == 1) { 1695 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 1696 } else if (!active_links) { 1697 smcr_lgr_set_type(lgr, SMC_LGR_NONE); 1698 smc_lgr_terminate_sched(lgr); 1699 } 1700 1701 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { 1702 /* trigger setup of asymm alt link */ 1703 smc_llc_add_link_local(lnk); 1704 } 1705 out: 1706 mutex_unlock(&lgr->llc_conf_mutex); 1707 kfree(qentry); 1708 } 1709 1710 static void smc_llc_delete_link_work(struct work_struct *work) 1711 { 1712 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1713 llc_del_link_work); 1714 1715 if (list_empty(&lgr->list)) { 1716 /* link group is terminating */ 1717 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1718 goto out; 1719 } 1720 1721 if (lgr->role == SMC_CLNT) 1722 smc_llc_process_cli_delete_link(lgr); 1723 else 1724 smc_llc_process_srv_delete_link(lgr); 1725 out: 1726 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1727 } 1728 1729 /* process a confirm_rkey request from peer, remote flow */ 1730 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 1731 { 1732 struct smc_llc_msg_confirm_rkey *llc; 1733 struct smc_llc_qentry *qentry; 1734 struct smc_link *link; 1735 int num_entries; 1736 int rk_idx; 1737 int i; 1738 1739 qentry = lgr->llc_flow_rmt.qentry; 1740 llc = &qentry->msg.confirm_rkey; 1741 link = qentry->link; 1742 1743 num_entries = llc->rtoken[0].num_rkeys; 1744 if (num_entries > SMC_LLC_RKEYS_PER_MSG) 1745 goto out_err; 1746 /* first rkey entry is for receiving link */ 1747 rk_idx = smc_rtoken_add(link, 1748 llc->rtoken[0].rmb_vaddr, 1749 llc->rtoken[0].rmb_key); 1750 if (rk_idx < 0) 1751 goto out_err; 1752 1753 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 1754 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 1755 llc->rtoken[i].rmb_vaddr, 1756 llc->rtoken[i].rmb_key); 1757 /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 1758 goto out; 1759 out_err: 1760 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1761 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 1762 out: 1763 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1764 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc)); 1765 smc_llc_send_message(link, &qentry->msg); 1766 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1767 } 1768 1769 /* process a delete_rkey request from peer, remote flow */ 1770 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 1771 { 1772 struct smc_llc_msg_delete_rkey *llc; 1773 struct smc_llc_qentry *qentry; 1774 struct smc_link *link; 1775 u8 err_mask = 0; 1776 int i, max; 1777 1778 qentry = lgr->llc_flow_rmt.qentry; 1779 llc = &qentry->msg.delete_rkey; 1780 link = qentry->link; 1781 1782 if (lgr->smc_version == SMC_V2) { 1783 struct smc_llc_msg_delete_rkey_v2 *llcv2; 1784 1785 memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc)); 1786 llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2; 1787 llcv2->num_inval_rkeys = 0; 1788 1789 max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); 1790 for (i = 0; i < max; i++) { 1791 if (smc_rtoken_delete(link, llcv2->rkey[i])) 1792 llcv2->num_inval_rkeys++; 1793 } 1794 memset(&llc->rkey[0], 0, sizeof(llc->rkey)); 1795 memset(&llc->reserved2, 0, sizeof(llc->reserved2)); 1796 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc)); 1797 if (llcv2->num_inval_rkeys) { 1798 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1799 llc->err_mask = llcv2->num_inval_rkeys; 1800 } 1801 goto finish; 1802 } 1803 1804 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 1805 for (i = 0; i < max; i++) { 1806 if (smc_rtoken_delete(link, llc->rkey[i])) 1807 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 1808 } 1809 if (err_mask) { 1810 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1811 llc->err_mask = err_mask; 1812 } 1813 finish: 1814 llc->hd.flags |= SMC_LLC_FLAG_RESP; 1815 smc_llc_send_message(link, &qentry->msg); 1816 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1817 } 1818 1819 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) 1820 { 1821 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: " 1822 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, 1823 lgr->net->net_cookie, type); 1824 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); 1825 smc_lgr_terminate_sched(lgr); 1826 } 1827 1828 /* flush the llc event queue */ 1829 static void smc_llc_event_flush(struct smc_link_group *lgr) 1830 { 1831 struct smc_llc_qentry *qentry, *q; 1832 1833 spin_lock_bh(&lgr->llc_event_q_lock); 1834 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 1835 list_del_init(&qentry->list); 1836 kfree(qentry); 1837 } 1838 spin_unlock_bh(&lgr->llc_event_q_lock); 1839 } 1840 1841 static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 1842 { 1843 union smc_llc_msg *llc = &qentry->msg; 1844 struct smc_link *link = qentry->link; 1845 struct smc_link_group *lgr = link->lgr; 1846 1847 if (!smc_link_usable(link)) 1848 goto out; 1849 1850 switch (llc->raw.hdr.common.llc_type) { 1851 case SMC_LLC_TEST_LINK: 1852 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 1853 smc_llc_send_message(link, llc); 1854 break; 1855 case SMC_LLC_ADD_LINK: 1856 if (list_empty(&lgr->list)) 1857 goto out; /* lgr is terminating */ 1858 if (lgr->role == SMC_CLNT) { 1859 if (smc_llc_is_local_add_link(llc)) { 1860 if (lgr->llc_flow_lcl.type == 1861 SMC_LLC_FLOW_ADD_LINK) 1862 break; /* add_link in progress */ 1863 if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1864 qentry)) { 1865 schedule_work(&lgr->llc_add_link_work); 1866 } 1867 return; 1868 } 1869 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1870 !lgr->llc_flow_lcl.qentry) { 1871 /* a flow is waiting for this message */ 1872 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1873 qentry); 1874 wake_up(&lgr->llc_msg_waiter); 1875 return; 1876 } 1877 if (lgr->llc_flow_lcl.type == 1878 SMC_LLC_FLOW_REQ_ADD_LINK) { 1879 /* server started add_link processing */ 1880 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; 1881 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1882 qentry); 1883 schedule_work(&lgr->llc_add_link_work); 1884 return; 1885 } 1886 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1887 schedule_work(&lgr->llc_add_link_work); 1888 } 1889 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1890 /* as smc server, handle client suggestion */ 1891 schedule_work(&lgr->llc_add_link_work); 1892 } 1893 return; 1894 case SMC_LLC_CONFIRM_LINK: 1895 case SMC_LLC_ADD_LINK_CONT: 1896 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 1897 /* a flow is waiting for this message */ 1898 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1899 wake_up(&lgr->llc_msg_waiter); 1900 return; 1901 } 1902 break; 1903 case SMC_LLC_DELETE_LINK: 1904 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1905 !lgr->llc_flow_lcl.qentry) { 1906 /* DEL LINK REQ during ADD LINK SEQ */ 1907 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1908 wake_up(&lgr->llc_msg_waiter); 1909 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1910 schedule_work(&lgr->llc_del_link_work); 1911 } 1912 return; 1913 case SMC_LLC_CONFIRM_RKEY: 1914 /* new request from remote, assign to remote flow */ 1915 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1916 /* process here, does not wait for more llc msgs */ 1917 smc_llc_rmt_conf_rkey(lgr); 1918 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1919 } 1920 return; 1921 case SMC_LLC_CONFIRM_RKEY_CONT: 1922 /* not used because max links is 3, and 3 rkeys fit into 1923 * one CONFIRM_RKEY message 1924 */ 1925 break; 1926 case SMC_LLC_DELETE_RKEY: 1927 /* new request from remote, assign to remote flow */ 1928 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1929 /* process here, does not wait for more llc msgs */ 1930 smc_llc_rmt_delete_rkey(lgr); 1931 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1932 } 1933 return; 1934 case SMC_LLC_REQ_ADD_LINK: 1935 /* handle response here, smc_llc_flow_stop() cannot be called 1936 * in tasklet context 1937 */ 1938 if (lgr->role == SMC_CLNT && 1939 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK && 1940 (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) { 1941 smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl); 1942 } else if (lgr->role == SMC_SERV) { 1943 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1944 /* as smc server, handle client suggestion */ 1945 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; 1946 schedule_work(&lgr->llc_add_link_work); 1947 } 1948 return; 1949 } 1950 break; 1951 default: 1952 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); 1953 break; 1954 } 1955 out: 1956 kfree(qentry); 1957 } 1958 1959 /* worker to process llc messages on the event queue */ 1960 static void smc_llc_event_work(struct work_struct *work) 1961 { 1962 struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1963 llc_event_work); 1964 struct smc_llc_qentry *qentry; 1965 1966 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 1967 qentry = lgr->delayed_event; 1968 lgr->delayed_event = NULL; 1969 if (smc_link_usable(qentry->link)) 1970 smc_llc_event_handler(qentry); 1971 else 1972 kfree(qentry); 1973 } 1974 1975 again: 1976 spin_lock_bh(&lgr->llc_event_q_lock); 1977 if (!list_empty(&lgr->llc_event_q)) { 1978 qentry = list_first_entry(&lgr->llc_event_q, 1979 struct smc_llc_qentry, list); 1980 list_del_init(&qentry->list); 1981 spin_unlock_bh(&lgr->llc_event_q_lock); 1982 smc_llc_event_handler(qentry); 1983 goto again; 1984 } 1985 spin_unlock_bh(&lgr->llc_event_q_lock); 1986 } 1987 1988 /* process llc responses in tasklet context */ 1989 static void smc_llc_rx_response(struct smc_link *link, 1990 struct smc_llc_qentry *qentry) 1991 { 1992 enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type; 1993 struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl; 1994 u8 llc_type = qentry->msg.raw.hdr.common.llc_type; 1995 1996 switch (llc_type) { 1997 case SMC_LLC_TEST_LINK: 1998 if (smc_link_active(link)) 1999 complete(&link->llc_testlink_resp); 2000 break; 2001 case SMC_LLC_ADD_LINK: 2002 case SMC_LLC_ADD_LINK_CONT: 2003 case SMC_LLC_CONFIRM_LINK: 2004 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry) 2005 break; /* drop out-of-flow response */ 2006 goto assign; 2007 case SMC_LLC_DELETE_LINK: 2008 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry) 2009 break; /* drop out-of-flow response */ 2010 goto assign; 2011 case SMC_LLC_CONFIRM_RKEY: 2012 case SMC_LLC_DELETE_RKEY: 2013 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry) 2014 break; /* drop out-of-flow response */ 2015 goto assign; 2016 case SMC_LLC_CONFIRM_RKEY_CONT: 2017 /* not used because max links is 3 */ 2018 break; 2019 default: 2020 smc_llc_protocol_violation(link->lgr, 2021 qentry->msg.raw.hdr.common.type); 2022 break; 2023 } 2024 kfree(qentry); 2025 return; 2026 assign: 2027 /* assign responses to the local flow, we requested them */ 2028 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 2029 wake_up(&link->lgr->llc_msg_waiter); 2030 } 2031 2032 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 2033 { 2034 struct smc_link_group *lgr = link->lgr; 2035 struct smc_llc_qentry *qentry; 2036 unsigned long flags; 2037 2038 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 2039 if (!qentry) 2040 return; 2041 qentry->link = link; 2042 INIT_LIST_HEAD(&qentry->list); 2043 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 2044 2045 /* process responses immediately */ 2046 if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) && 2047 llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) { 2048 smc_llc_rx_response(link, qentry); 2049 return; 2050 } 2051 2052 /* add requests to event queue */ 2053 spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 2054 list_add_tail(&qentry->list, &lgr->llc_event_q); 2055 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 2056 queue_work(system_highpri_wq, &lgr->llc_event_work); 2057 } 2058 2059 /* copy received msg and add it to the event queue */ 2060 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 2061 { 2062 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 2063 union smc_llc_msg *llc = buf; 2064 2065 if (wc->byte_len < sizeof(*llc)) 2066 return; /* short message */ 2067 if (!llc->raw.hdr.common.llc_version) { 2068 if (llc->raw.hdr.length != sizeof(*llc)) 2069 return; /* invalid message */ 2070 } else { 2071 if (llc->raw.hdr.length_v2 < sizeof(*llc)) 2072 return; /* invalid message */ 2073 } 2074 2075 smc_llc_enqueue(link, llc); 2076 } 2077 2078 /***************************** worker, utils *********************************/ 2079 2080 static void smc_llc_testlink_work(struct work_struct *work) 2081 { 2082 struct smc_link *link = container_of(to_delayed_work(work), 2083 struct smc_link, llc_testlink_wrk); 2084 unsigned long next_interval; 2085 unsigned long expire_time; 2086 u8 user_data[16] = { 0 }; 2087 int rc; 2088 2089 if (!smc_link_active(link)) 2090 return; /* don't reschedule worker */ 2091 expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 2092 if (time_is_after_jiffies(expire_time)) { 2093 next_interval = expire_time - jiffies; 2094 goto out; 2095 } 2096 reinit_completion(&link->llc_testlink_resp); 2097 smc_llc_send_test_link(link, user_data); 2098 /* receive TEST LINK response over RoCE fabric */ 2099 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 2100 SMC_LLC_WAIT_TIME); 2101 if (!smc_link_active(link)) 2102 return; /* link state changed */ 2103 if (rc <= 0) { 2104 smcr_link_down_cond_sched(link); 2105 return; 2106 } 2107 next_interval = link->llc_testlink_time; 2108 out: 2109 schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 2110 } 2111 2112 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 2113 { 2114 struct net *net = sock_net(smc->clcsock->sk); 2115 2116 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 2117 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 2118 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); 2119 INIT_LIST_HEAD(&lgr->llc_event_q); 2120 spin_lock_init(&lgr->llc_event_q_lock); 2121 spin_lock_init(&lgr->llc_flow_lock); 2122 init_waitqueue_head(&lgr->llc_flow_waiter); 2123 init_waitqueue_head(&lgr->llc_msg_waiter); 2124 mutex_init(&lgr->llc_conf_mutex); 2125 lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; 2126 } 2127 2128 /* called after lgr was removed from lgr_list */ 2129 void smc_llc_lgr_clear(struct smc_link_group *lgr) 2130 { 2131 smc_llc_event_flush(lgr); 2132 wake_up_all(&lgr->llc_flow_waiter); 2133 wake_up_all(&lgr->llc_msg_waiter); 2134 cancel_work_sync(&lgr->llc_event_work); 2135 cancel_work_sync(&lgr->llc_add_link_work); 2136 cancel_work_sync(&lgr->llc_del_link_work); 2137 if (lgr->delayed_event) { 2138 kfree(lgr->delayed_event); 2139 lgr->delayed_event = NULL; 2140 } 2141 } 2142 2143 int smc_llc_link_init(struct smc_link *link) 2144 { 2145 init_completion(&link->llc_testlink_resp); 2146 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 2147 return 0; 2148 } 2149 2150 void smc_llc_link_active(struct smc_link *link) 2151 { 2152 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, " 2153 "peerid %*phN, ibdev %s, ibport %d\n", 2154 SMC_LGR_ID_SIZE, &link->lgr->id, 2155 link->lgr->net->net_cookie, 2156 SMC_LGR_ID_SIZE, &link->link_uid, 2157 SMC_LGR_ID_SIZE, &link->peer_link_uid, 2158 link->smcibdev->ibdev->name, link->ibport); 2159 link->state = SMC_LNK_ACTIVE; 2160 if (link->lgr->llc_testlink_time) { 2161 link->llc_testlink_time = link->lgr->llc_testlink_time; 2162 schedule_delayed_work(&link->llc_testlink_wrk, 2163 link->llc_testlink_time); 2164 } 2165 } 2166 2167 /* called in worker context */ 2168 void smc_llc_link_clear(struct smc_link *link, bool log) 2169 { 2170 if (log) 2171 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN" 2172 ", peerid %*phN, ibdev %s, ibport %d\n", 2173 SMC_LGR_ID_SIZE, &link->lgr->id, 2174 link->lgr->net->net_cookie, 2175 SMC_LGR_ID_SIZE, &link->link_uid, 2176 SMC_LGR_ID_SIZE, &link->peer_link_uid, 2177 link->smcibdev->ibdev->name, link->ibport); 2178 complete(&link->llc_testlink_resp); 2179 cancel_delayed_work_sync(&link->llc_testlink_wrk); 2180 } 2181 2182 /* register a new rtoken at the remote peer (for all links) */ 2183 int smc_llc_do_confirm_rkey(struct smc_link *send_link, 2184 struct smc_buf_desc *rmb_desc) 2185 { 2186 struct smc_link_group *lgr = send_link->lgr; 2187 struct smc_llc_qentry *qentry = NULL; 2188 int rc = 0; 2189 2190 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 2191 if (rc) 2192 goto out; 2193 /* receive CONFIRM RKEY response from server over RoCE fabric */ 2194 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 2195 SMC_LLC_CONFIRM_RKEY); 2196 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 2197 rc = -EFAULT; 2198 out: 2199 if (qentry) 2200 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 2201 return rc; 2202 } 2203 2204 /* unregister an rtoken at the remote peer */ 2205 int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 2206 struct smc_buf_desc *rmb_desc) 2207 { 2208 struct smc_llc_qentry *qentry = NULL; 2209 struct smc_link *send_link; 2210 int rc = 0; 2211 2212 send_link = smc_llc_usable_link(lgr); 2213 if (!send_link) 2214 return -ENOLINK; 2215 2216 /* protected by llc_flow control */ 2217 rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 2218 if (rc) 2219 goto out; 2220 /* receive DELETE RKEY response from server over RoCE fabric */ 2221 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 2222 SMC_LLC_DELETE_RKEY); 2223 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 2224 rc = -EFAULT; 2225 out: 2226 if (qentry) 2227 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 2228 return rc; 2229 } 2230 2231 void smc_llc_link_set_uid(struct smc_link *link) 2232 { 2233 __be32 link_uid; 2234 2235 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); 2236 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); 2237 } 2238 2239 /* save peers link user id, used for debug purposes */ 2240 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) 2241 { 2242 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, 2243 SMC_LGR_ID_SIZE); 2244 } 2245 2246 /* evaluate confirm link request or response */ 2247 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 2248 enum smc_llc_reqresp type) 2249 { 2250 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ 2251 qentry->link->link_id = qentry->msg.confirm_link.link_num; 2252 smc_llc_link_set_uid(qentry->link); 2253 } 2254 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 2255 return -ENOTSUPP; 2256 return 0; 2257 } 2258 2259 /***************************** init, exit, misc ******************************/ 2260 2261 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 2262 { 2263 .handler = smc_llc_rx_handler, 2264 .type = SMC_LLC_CONFIRM_LINK 2265 }, 2266 { 2267 .handler = smc_llc_rx_handler, 2268 .type = SMC_LLC_TEST_LINK 2269 }, 2270 { 2271 .handler = smc_llc_rx_handler, 2272 .type = SMC_LLC_ADD_LINK 2273 }, 2274 { 2275 .handler = smc_llc_rx_handler, 2276 .type = SMC_LLC_ADD_LINK_CONT 2277 }, 2278 { 2279 .handler = smc_llc_rx_handler, 2280 .type = SMC_LLC_DELETE_LINK 2281 }, 2282 { 2283 .handler = smc_llc_rx_handler, 2284 .type = SMC_LLC_CONFIRM_RKEY 2285 }, 2286 { 2287 .handler = smc_llc_rx_handler, 2288 .type = SMC_LLC_CONFIRM_RKEY_CONT 2289 }, 2290 { 2291 .handler = smc_llc_rx_handler, 2292 .type = SMC_LLC_DELETE_RKEY 2293 }, 2294 /* V2 types */ 2295 { 2296 .handler = smc_llc_rx_handler, 2297 .type = SMC_LLC_CONFIRM_LINK_V2 2298 }, 2299 { 2300 .handler = smc_llc_rx_handler, 2301 .type = SMC_LLC_TEST_LINK_V2 2302 }, 2303 { 2304 .handler = smc_llc_rx_handler, 2305 .type = SMC_LLC_ADD_LINK_V2 2306 }, 2307 { 2308 .handler = smc_llc_rx_handler, 2309 .type = SMC_LLC_DELETE_LINK_V2 2310 }, 2311 { 2312 .handler = smc_llc_rx_handler, 2313 .type = SMC_LLC_REQ_ADD_LINK_V2 2314 }, 2315 { 2316 .handler = smc_llc_rx_handler, 2317 .type = SMC_LLC_CONFIRM_RKEY_V2 2318 }, 2319 { 2320 .handler = smc_llc_rx_handler, 2321 .type = SMC_LLC_DELETE_RKEY_V2 2322 }, 2323 { 2324 .handler = NULL, 2325 } 2326 }; 2327 2328 int __init smc_llc_init(void) 2329 { 2330 struct smc_wr_rx_handler *handler; 2331 int rc = 0; 2332 2333 for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 2334 INIT_HLIST_NODE(&handler->list); 2335 rc = smc_wr_rx_register_handler(handler); 2336 if (rc) 2337 break; 2338 } 2339 return rc; 2340 } 2341