1 /* 2 * Shared Memory Communications over RDMA (SMC-R) and RoCE 3 * 4 * Basic Transport Functions exploiting Infiniband API 5 * 6 * Copyright IBM Corp. 2016 7 * 8 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 9 */ 10 11 #include <linux/socket.h> 12 #include <linux/if_vlan.h> 13 #include <linux/random.h> 14 #include <linux/workqueue.h> 15 #include <net/tcp.h> 16 #include <net/sock.h> 17 #include <rdma/ib_verbs.h> 18 19 #include "smc.h" 20 #include "smc_clc.h" 21 #include "smc_core.h" 22 #include "smc_ib.h" 23 #include "smc_wr.h" 24 #include "smc_llc.h" 25 #include "smc_cdc.h" 26 #include "smc_close.h" 27 28 #define SMC_LGR_NUM_INCR 256 29 #define SMC_LGR_FREE_DELAY (600 * HZ) 30 31 static u32 smc_lgr_num; /* unique link group number */ 32 33 /* Register connection's alert token in our lookup structure. 34 * To use rbtrees we have to implement our own insert core. 35 * Requires @conns_lock 36 * @smc connection to register 37 * Returns 0 on success, != otherwise. 38 */ 39 static void smc_lgr_add_alert_token(struct smc_connection *conn) 40 { 41 struct rb_node **link, *parent = NULL; 42 u32 token = conn->alert_token_local; 43 44 link = &conn->lgr->conns_all.rb_node; 45 while (*link) { 46 struct smc_connection *cur = rb_entry(*link, 47 struct smc_connection, alert_node); 48 49 parent = *link; 50 if (cur->alert_token_local > token) 51 link = &parent->rb_left; 52 else 53 link = &parent->rb_right; 54 } 55 /* Put the new node there */ 56 rb_link_node(&conn->alert_node, parent, link); 57 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all); 58 } 59 60 /* Register connection in link group by assigning an alert token 61 * registered in a search tree. 62 * Requires @conns_lock 63 * Note that '0' is a reserved value and not assigned. 64 */ 65 static void smc_lgr_register_conn(struct smc_connection *conn) 66 { 67 struct smc_sock *smc = container_of(conn, struct smc_sock, conn); 68 static atomic_t nexttoken = ATOMIC_INIT(0); 69 70 /* find a new alert_token_local value not yet used by some connection 71 * in this link group 72 */ 73 sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */ 74 while (!conn->alert_token_local) { 75 conn->alert_token_local = atomic_inc_return(&nexttoken); 76 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr)) 77 conn->alert_token_local = 0; 78 } 79 smc_lgr_add_alert_token(conn); 80 conn->lgr->conns_num++; 81 } 82 83 /* Unregister connection and reset the alert token of the given connection< 84 */ 85 static void __smc_lgr_unregister_conn(struct smc_connection *conn) 86 { 87 struct smc_sock *smc = container_of(conn, struct smc_sock, conn); 88 struct smc_link_group *lgr = conn->lgr; 89 90 rb_erase(&conn->alert_node, &lgr->conns_all); 91 lgr->conns_num--; 92 conn->alert_token_local = 0; 93 conn->lgr = NULL; 94 sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */ 95 } 96 97 /* Unregister connection and trigger lgr freeing if applicable 98 */ 99 static void smc_lgr_unregister_conn(struct smc_connection *conn) 100 { 101 struct smc_link_group *lgr = conn->lgr; 102 int reduced = 0; 103 104 write_lock_bh(&lgr->conns_lock); 105 if (conn->alert_token_local) { 106 reduced = 1; 107 __smc_lgr_unregister_conn(conn); 108 } 109 write_unlock_bh(&lgr->conns_lock); 110 if (reduced && !lgr->conns_num) 111 schedule_delayed_work(&lgr->free_work, SMC_LGR_FREE_DELAY); 112 } 113 114 static void smc_lgr_free_work(struct work_struct *work) 115 { 116 struct smc_link_group *lgr = container_of(to_delayed_work(work), 117 struct smc_link_group, 118 free_work); 119 bool conns; 120 121 spin_lock_bh(&smc_lgr_list.lock); 122 read_lock_bh(&lgr->conns_lock); 123 conns = RB_EMPTY_ROOT(&lgr->conns_all); 124 read_unlock_bh(&lgr->conns_lock); 125 if (!conns) { /* number of lgr connections is no longer zero */ 126 spin_unlock_bh(&smc_lgr_list.lock); 127 return; 128 } 129 list_del_init(&lgr->list); /* remove from smc_lgr_list */ 130 spin_unlock_bh(&smc_lgr_list.lock); 131 smc_lgr_free(lgr); 132 } 133 134 /* create a new SMC link group */ 135 static int smc_lgr_create(struct smc_sock *smc, __be32 peer_in_addr, 136 struct smc_ib_device *smcibdev, u8 ibport, 137 char *peer_systemid, unsigned short vlan_id) 138 { 139 struct smc_link_group *lgr; 140 struct smc_link *lnk; 141 u8 rndvec[3]; 142 int rc = 0; 143 int i; 144 145 lgr = kzalloc(sizeof(*lgr), GFP_KERNEL); 146 if (!lgr) { 147 rc = -ENOMEM; 148 goto out; 149 } 150 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; 151 lgr->sync_err = false; 152 lgr->daddr = peer_in_addr; 153 memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN); 154 lgr->vlan_id = vlan_id; 155 rwlock_init(&lgr->sndbufs_lock); 156 rwlock_init(&lgr->rmbs_lock); 157 for (i = 0; i < SMC_RMBE_SIZES; i++) { 158 INIT_LIST_HEAD(&lgr->sndbufs[i]); 159 INIT_LIST_HEAD(&lgr->rmbs[i]); 160 } 161 smc_lgr_num += SMC_LGR_NUM_INCR; 162 memcpy(&lgr->id, (u8 *)&smc_lgr_num, SMC_LGR_ID_SIZE); 163 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work); 164 lgr->conns_all = RB_ROOT; 165 166 lnk = &lgr->lnk[SMC_SINGLE_LINK]; 167 /* initialize link */ 168 lnk->smcibdev = smcibdev; 169 lnk->ibport = ibport; 170 lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu; 171 if (!smcibdev->initialized) 172 smc_ib_setup_per_ibdev(smcibdev); 173 get_random_bytes(rndvec, sizeof(rndvec)); 174 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + (rndvec[2] << 16); 175 rc = smc_wr_alloc_link_mem(lnk); 176 if (rc) 177 goto free_lgr; 178 rc = smc_ib_create_protection_domain(lnk); 179 if (rc) 180 goto free_link_mem; 181 rc = smc_ib_create_queue_pair(lnk); 182 if (rc) 183 goto dealloc_pd; 184 rc = smc_wr_create_link(lnk); 185 if (rc) 186 goto destroy_qp; 187 init_completion(&lnk->llc_confirm); 188 init_completion(&lnk->llc_confirm_resp); 189 190 smc->conn.lgr = lgr; 191 rwlock_init(&lgr->conns_lock); 192 spin_lock_bh(&smc_lgr_list.lock); 193 list_add(&lgr->list, &smc_lgr_list.list); 194 spin_unlock_bh(&smc_lgr_list.lock); 195 return 0; 196 197 destroy_qp: 198 smc_ib_destroy_queue_pair(lnk); 199 dealloc_pd: 200 smc_ib_dealloc_protection_domain(lnk); 201 free_link_mem: 202 smc_wr_free_link_mem(lnk); 203 free_lgr: 204 kfree(lgr); 205 out: 206 return rc; 207 } 208 209 static void smc_buf_unuse(struct smc_connection *conn) 210 { 211 if (conn->sndbuf_desc) { 212 conn->sndbuf_desc->used = 0; 213 conn->sndbuf_size = 0; 214 } 215 if (conn->rmb_desc) { 216 conn->rmb_desc->reused = true; 217 conn->rmb_desc->used = 0; 218 conn->rmbe_size = 0; 219 } 220 } 221 222 /* remove a finished connection from its link group */ 223 void smc_conn_free(struct smc_connection *conn) 224 { 225 struct smc_link_group *lgr = conn->lgr; 226 227 if (!lgr) 228 return; 229 smc_cdc_tx_dismiss_slots(conn); 230 smc_lgr_unregister_conn(conn); 231 smc_buf_unuse(conn); 232 } 233 234 static void smc_link_clear(struct smc_link *lnk) 235 { 236 lnk->peer_qpn = 0; 237 smc_ib_modify_qp_reset(lnk); 238 smc_wr_free_link(lnk); 239 smc_ib_destroy_queue_pair(lnk); 240 smc_ib_dealloc_protection_domain(lnk); 241 smc_wr_free_link_mem(lnk); 242 } 243 244 static void smc_buf_free(struct smc_buf_desc *buf_desc, struct smc_link *lnk, 245 bool is_rmb) 246 { 247 if (is_rmb) { 248 if (buf_desc->mr_rx[SMC_SINGLE_LINK]) 249 smc_ib_put_memory_region( 250 buf_desc->mr_rx[SMC_SINGLE_LINK]); 251 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, 252 DMA_FROM_DEVICE); 253 } else { 254 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc, 255 DMA_TO_DEVICE); 256 } 257 sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]); 258 if (buf_desc->cpu_addr) 259 free_pages((unsigned long)buf_desc->cpu_addr, buf_desc->order); 260 kfree(buf_desc); 261 } 262 263 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb) 264 { 265 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; 266 struct smc_buf_desc *buf_desc, *bf_desc; 267 struct list_head *buf_list; 268 int i; 269 270 for (i = 0; i < SMC_RMBE_SIZES; i++) { 271 if (is_rmb) 272 buf_list = &lgr->rmbs[i]; 273 else 274 buf_list = &lgr->sndbufs[i]; 275 list_for_each_entry_safe(buf_desc, bf_desc, buf_list, 276 list) { 277 list_del(&buf_desc->list); 278 smc_buf_free(buf_desc, lnk, is_rmb); 279 } 280 } 281 } 282 283 static void smc_lgr_free_bufs(struct smc_link_group *lgr) 284 { 285 /* free send buffers */ 286 __smc_lgr_free_bufs(lgr, false); 287 /* free rmbs */ 288 __smc_lgr_free_bufs(lgr, true); 289 } 290 291 /* remove a link group */ 292 void smc_lgr_free(struct smc_link_group *lgr) 293 { 294 smc_lgr_free_bufs(lgr); 295 smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]); 296 kfree(lgr); 297 } 298 299 /* terminate linkgroup abnormally */ 300 void smc_lgr_terminate(struct smc_link_group *lgr) 301 { 302 struct smc_connection *conn; 303 struct smc_sock *smc; 304 struct rb_node *node; 305 306 spin_lock_bh(&smc_lgr_list.lock); 307 if (list_empty(&lgr->list)) { 308 /* termination already triggered */ 309 spin_unlock_bh(&smc_lgr_list.lock); 310 return; 311 } 312 /* do not use this link group for new connections */ 313 list_del_init(&lgr->list); 314 spin_unlock_bh(&smc_lgr_list.lock); 315 316 write_lock_bh(&lgr->conns_lock); 317 node = rb_first(&lgr->conns_all); 318 while (node) { 319 conn = rb_entry(node, struct smc_connection, alert_node); 320 smc = container_of(conn, struct smc_sock, conn); 321 sock_hold(&smc->sk); 322 __smc_lgr_unregister_conn(conn); 323 schedule_work(&conn->close_work); 324 sock_put(&smc->sk); 325 node = rb_first(&lgr->conns_all); 326 } 327 write_unlock_bh(&lgr->conns_lock); 328 } 329 330 /* Determine vlan of internal TCP socket. 331 * @vlan_id: address to store the determined vlan id into 332 */ 333 static int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id) 334 { 335 struct dst_entry *dst = sk_dst_get(clcsock->sk); 336 int rc = 0; 337 338 *vlan_id = 0; 339 if (!dst) { 340 rc = -ENOTCONN; 341 goto out; 342 } 343 if (!dst->dev) { 344 rc = -ENODEV; 345 goto out_rel; 346 } 347 348 if (is_vlan_dev(dst->dev)) 349 *vlan_id = vlan_dev_vlan_id(dst->dev); 350 351 out_rel: 352 dst_release(dst); 353 out: 354 return rc; 355 } 356 357 /* determine the link gid matching the vlan id of the link group */ 358 static int smc_link_determine_gid(struct smc_link_group *lgr) 359 { 360 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK]; 361 struct ib_gid_attr gattr; 362 union ib_gid gid; 363 int i; 364 365 if (!lgr->vlan_id) { 366 lnk->gid = lnk->smcibdev->gid[lnk->ibport - 1]; 367 return 0; 368 } 369 370 for (i = 0; i < lnk->smcibdev->pattr[lnk->ibport - 1].gid_tbl_len; 371 i++) { 372 if (ib_query_gid(lnk->smcibdev->ibdev, lnk->ibport, i, &gid, 373 &gattr)) 374 continue; 375 if (gattr.ndev && 376 (vlan_dev_vlan_id(gattr.ndev) == lgr->vlan_id)) { 377 lnk->gid = gid; 378 return 0; 379 } 380 } 381 return -ENODEV; 382 } 383 384 /* create a new SMC connection (and a new link group if necessary) */ 385 int smc_conn_create(struct smc_sock *smc, __be32 peer_in_addr, 386 struct smc_ib_device *smcibdev, u8 ibport, 387 struct smc_clc_msg_local *lcl, int srv_first_contact) 388 { 389 struct smc_connection *conn = &smc->conn; 390 struct smc_link_group *lgr; 391 unsigned short vlan_id; 392 enum smc_lgr_role role; 393 int local_contact = SMC_FIRST_CONTACT; 394 int rc = 0; 395 396 role = smc->listen_smc ? SMC_SERV : SMC_CLNT; 397 rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id); 398 if (rc) 399 return rc; 400 401 if ((role == SMC_CLNT) && srv_first_contact) 402 /* create new link group as well */ 403 goto create; 404 405 /* determine if an existing link group can be reused */ 406 spin_lock_bh(&smc_lgr_list.lock); 407 list_for_each_entry(lgr, &smc_lgr_list.list, list) { 408 write_lock_bh(&lgr->conns_lock); 409 if (!memcmp(lgr->peer_systemid, lcl->id_for_peer, 410 SMC_SYSTEMID_LEN) && 411 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid, 412 SMC_GID_SIZE) && 413 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac, 414 sizeof(lcl->mac)) && 415 !lgr->sync_err && 416 (lgr->role == role) && 417 (lgr->vlan_id == vlan_id) && 418 ((role == SMC_CLNT) || 419 (lgr->conns_num < SMC_RMBS_PER_LGR_MAX))) { 420 /* link group found */ 421 local_contact = SMC_REUSE_CONTACT; 422 conn->lgr = lgr; 423 smc_lgr_register_conn(conn); /* add smc conn to lgr */ 424 write_unlock_bh(&lgr->conns_lock); 425 break; 426 } 427 write_unlock_bh(&lgr->conns_lock); 428 } 429 spin_unlock_bh(&smc_lgr_list.lock); 430 431 if (role == SMC_CLNT && !srv_first_contact && 432 (local_contact == SMC_FIRST_CONTACT)) { 433 /* Server reuses a link group, but Client wants to start 434 * a new one 435 * send out_of_sync decline, reason synchr. error 436 */ 437 return -ENOLINK; 438 } 439 440 create: 441 if (local_contact == SMC_FIRST_CONTACT) { 442 rc = smc_lgr_create(smc, peer_in_addr, smcibdev, ibport, 443 lcl->id_for_peer, vlan_id); 444 if (rc) 445 goto out; 446 smc_lgr_register_conn(conn); /* add smc conn to lgr */ 447 rc = smc_link_determine_gid(conn->lgr); 448 } 449 conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE; 450 conn->local_tx_ctrl.len = sizeof(struct smc_cdc_msg); 451 #ifndef KERNEL_HAS_ATOMIC64 452 spin_lock_init(&conn->acurs_lock); 453 #endif 454 455 out: 456 return rc ? rc : local_contact; 457 } 458 459 /* try to reuse a sndbuf or rmb description slot for a certain 460 * buffer size; if not available, return NULL 461 */ 462 static inline 463 struct smc_buf_desc *smc_buf_get_slot(struct smc_link_group *lgr, 464 int compressed_bufsize, 465 rwlock_t *lock, 466 struct list_head *buf_list) 467 { 468 struct smc_buf_desc *buf_slot; 469 470 read_lock_bh(lock); 471 list_for_each_entry(buf_slot, buf_list, list) { 472 if (cmpxchg(&buf_slot->used, 0, 1) == 0) { 473 read_unlock_bh(lock); 474 return buf_slot; 475 } 476 } 477 read_unlock_bh(lock); 478 return NULL; 479 } 480 481 /* one of the conditions for announcing a receiver's current window size is 482 * that it "results in a minimum increase in the window size of 10% of the 483 * receive buffer space" [RFC7609] 484 */ 485 static inline int smc_rmb_wnd_update_limit(int rmbe_size) 486 { 487 return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2); 488 } 489 490 static struct smc_buf_desc *smc_new_buf_create(struct smc_link_group *lgr, 491 bool is_rmb, int bufsize) 492 { 493 struct smc_buf_desc *buf_desc; 494 struct smc_link *lnk; 495 int rc; 496 497 /* try to alloc a new buffer */ 498 buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL); 499 if (!buf_desc) 500 return ERR_PTR(-ENOMEM); 501 502 buf_desc->cpu_addr = 503 (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | 504 __GFP_NOMEMALLOC | 505 __GFP_NORETRY | __GFP_ZERO, 506 get_order(bufsize)); 507 if (!buf_desc->cpu_addr) { 508 kfree(buf_desc); 509 return ERR_PTR(-EAGAIN); 510 } 511 buf_desc->order = get_order(bufsize); 512 513 /* build the sg table from the pages */ 514 lnk = &lgr->lnk[SMC_SINGLE_LINK]; 515 rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1, 516 GFP_KERNEL); 517 if (rc) { 518 smc_buf_free(buf_desc, lnk, is_rmb); 519 return ERR_PTR(rc); 520 } 521 sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl, 522 buf_desc->cpu_addr, bufsize); 523 524 /* map sg table to DMA address */ 525 rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc, 526 is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); 527 /* SMC protocol depends on mapping to one DMA address only */ 528 if (rc != 1) { 529 smc_buf_free(buf_desc, lnk, is_rmb); 530 return ERR_PTR(-EAGAIN); 531 } 532 533 /* create a new memory region for the RMB */ 534 if (is_rmb) { 535 rc = smc_ib_get_memory_region(lnk->roce_pd, 536 IB_ACCESS_REMOTE_WRITE | 537 IB_ACCESS_LOCAL_WRITE, 538 buf_desc); 539 if (rc) { 540 smc_buf_free(buf_desc, lnk, is_rmb); 541 return ERR_PTR(rc); 542 } 543 } 544 545 return buf_desc; 546 } 547 548 static int __smc_buf_create(struct smc_sock *smc, bool is_rmb) 549 { 550 struct smc_connection *conn = &smc->conn; 551 struct smc_link_group *lgr = conn->lgr; 552 struct smc_buf_desc *buf_desc = NULL; 553 struct list_head *buf_list; 554 int bufsize, bufsize_short; 555 int sk_buf_size; 556 rwlock_t *lock; 557 558 if (is_rmb) 559 /* use socket recv buffer size (w/o overhead) as start value */ 560 sk_buf_size = smc->sk.sk_rcvbuf / 2; 561 else 562 /* use socket send buffer size (w/o overhead) as start value */ 563 sk_buf_size = smc->sk.sk_sndbuf / 2; 564 565 for (bufsize_short = smc_compress_bufsize(smc->sk.sk_sndbuf / 2); 566 bufsize_short >= 0; bufsize_short--) { 567 568 if (is_rmb) { 569 lock = &lgr->rmbs_lock; 570 buf_list = &lgr->rmbs[bufsize_short]; 571 } else { 572 lock = &lgr->sndbufs_lock; 573 buf_list = &lgr->sndbufs[bufsize_short]; 574 } 575 bufsize = smc_uncompress_bufsize(bufsize_short); 576 if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC) 577 continue; 578 579 /* check for reusable slot in the link group */ 580 buf_desc = smc_buf_get_slot(lgr, bufsize_short, lock, buf_list); 581 if (buf_desc) { 582 memset(buf_desc->cpu_addr, 0, bufsize); 583 break; /* found reusable slot */ 584 } 585 586 buf_desc = smc_new_buf_create(lgr, is_rmb, bufsize); 587 if (PTR_ERR(buf_desc) == -ENOMEM) 588 break; 589 if (IS_ERR(buf_desc)) 590 continue; 591 592 buf_desc->used = 1; 593 write_lock_bh(lock); 594 list_add(&buf_desc->list, buf_list); 595 write_unlock_bh(lock); 596 break; /* found */ 597 } 598 599 if (IS_ERR(buf_desc)) 600 return -ENOMEM; 601 602 if (is_rmb) { 603 conn->rmb_desc = buf_desc; 604 conn->rmbe_size = bufsize; 605 conn->rmbe_size_short = bufsize_short; 606 smc->sk.sk_rcvbuf = bufsize * 2; 607 atomic_set(&conn->bytes_to_rcv, 0); 608 conn->rmbe_update_limit = smc_rmb_wnd_update_limit(bufsize); 609 } else { 610 conn->sndbuf_desc = buf_desc; 611 conn->sndbuf_size = bufsize; 612 smc->sk.sk_sndbuf = bufsize * 2; 613 atomic_set(&conn->sndbuf_space, bufsize); 614 } 615 return 0; 616 } 617 618 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn) 619 { 620 struct smc_link_group *lgr = conn->lgr; 621 622 smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 623 conn->sndbuf_desc, DMA_TO_DEVICE); 624 } 625 626 void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn) 627 { 628 struct smc_link_group *lgr = conn->lgr; 629 630 smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 631 conn->sndbuf_desc, DMA_TO_DEVICE); 632 } 633 634 void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn) 635 { 636 struct smc_link_group *lgr = conn->lgr; 637 638 smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 639 conn->rmb_desc, DMA_FROM_DEVICE); 640 } 641 642 void smc_rmb_sync_sg_for_device(struct smc_connection *conn) 643 { 644 struct smc_link_group *lgr = conn->lgr; 645 646 smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev, 647 conn->rmb_desc, DMA_FROM_DEVICE); 648 } 649 650 /* create the send and receive buffer for an SMC socket; 651 * receive buffers are called RMBs; 652 * (even though the SMC protocol allows more than one RMB-element per RMB, 653 * the Linux implementation uses just one RMB-element per RMB, i.e. uses an 654 * extra RMB for every connection in a link group 655 */ 656 int smc_buf_create(struct smc_sock *smc) 657 { 658 int rc; 659 660 /* create send buffer */ 661 rc = __smc_buf_create(smc, false); 662 if (rc) 663 return rc; 664 /* create rmb */ 665 rc = __smc_buf_create(smc, true); 666 if (rc) 667 smc_buf_free(smc->conn.sndbuf_desc, 668 &smc->conn.lgr->lnk[SMC_SINGLE_LINK], false); 669 return rc; 670 } 671 672 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr) 673 { 674 int i; 675 676 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) { 677 if (!test_and_set_bit(i, lgr->rtokens_used_mask)) 678 return i; 679 } 680 return -ENOSPC; 681 } 682 683 /* save rkey and dma_addr received from peer during clc handshake */ 684 int smc_rmb_rtoken_handling(struct smc_connection *conn, 685 struct smc_clc_msg_accept_confirm *clc) 686 { 687 u64 dma_addr = be64_to_cpu(clc->rmb_dma_addr); 688 struct smc_link_group *lgr = conn->lgr; 689 u32 rkey = ntohl(clc->rmb_rkey); 690 int i; 691 692 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { 693 if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) && 694 (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) && 695 test_bit(i, lgr->rtokens_used_mask)) { 696 conn->rtoken_idx = i; 697 return 0; 698 } 699 } 700 conn->rtoken_idx = smc_rmb_reserve_rtoken_idx(lgr); 701 if (conn->rtoken_idx < 0) 702 return conn->rtoken_idx; 703 lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].rkey = rkey; 704 lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].dma_addr = dma_addr; 705 return 0; 706 } 707