1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Work Requests exploiting Infiniband API 6 * 7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively 8 * are submitted to either RC SQ or RC RQ respectively 9 * (reliably connected send/receive queue) 10 * and become work queue entries (WQEs). 11 * While an SQ WR/WQE is pending, we track it until transmission completion. 12 * Through a send or receive completion queue (CQ) respectively, 13 * we get completion queue entries (CQEs) [aka work completions (WCs)]. 14 * Since the CQ callback is called from IRQ context, we split work by using 15 * bottom halves implemented by tasklets. 16 * 17 * SMC uses this to exchange LLC (link layer control) 18 * and CDC (connection data control) messages. 19 * 20 * Copyright IBM Corp. 2016 21 * 22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> 23 */ 24 25 #include <linux/atomic.h> 26 #include <linux/hashtable.h> 27 #include <linux/wait.h> 28 #include <rdma/ib_verbs.h> 29 #include <asm/div64.h> 30 31 #include "smc.h" 32 #include "smc_wr.h" 33 34 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */ 35 36 #define SMC_WR_RX_HASH_BITS 4 37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS); 38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock); 39 40 struct smc_wr_tx_pend { /* control data for a pending send request */ 41 u64 wr_id; /* work request id sent */ 42 smc_wr_tx_handler handler; 43 enum ib_wc_status wc_status; /* CQE status */ 44 struct smc_link *link; 45 u32 idx; 46 struct smc_wr_tx_pend_priv priv; 47 }; 48 49 /******************************** send queue *********************************/ 50 51 /*------------------------------- completion --------------------------------*/ 52 53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) 54 { 55 u32 i; 56 57 for (i = 0; i < link->wr_tx_cnt; i++) { 58 if (link->wr_tx_pends[i].wr_id == wr_id) 59 return i; 60 } 61 return link->wr_tx_cnt; 62 } 63 64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc) 65 { 66 struct smc_wr_tx_pend pnd_snd; 67 struct smc_link *link; 68 u32 pnd_snd_idx; 69 int i; 70 71 link = wc->qp->qp_context; 72 73 if (wc->opcode == IB_WC_REG_MR) { 74 if (wc->status) 75 link->wr_reg_state = FAILED; 76 else 77 link->wr_reg_state = CONFIRMED; 78 wake_up(&link->wr_reg_wait); 79 return; 80 } 81 82 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); 83 if (pnd_snd_idx == link->wr_tx_cnt) 84 return; 85 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; 86 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd)); 87 /* clear the full struct smc_wr_tx_pend including .priv */ 88 memset(&link->wr_tx_pends[pnd_snd_idx], 0, 89 sizeof(link->wr_tx_pends[pnd_snd_idx])); 90 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, 91 sizeof(link->wr_tx_bufs[pnd_snd_idx])); 92 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) 93 return; 94 if (wc->status) { 95 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 96 /* clear full struct smc_wr_tx_pend including .priv */ 97 memset(&link->wr_tx_pends[i], 0, 98 sizeof(link->wr_tx_pends[i])); 99 memset(&link->wr_tx_bufs[i], 0, 100 sizeof(link->wr_tx_bufs[i])); 101 clear_bit(i, link->wr_tx_mask); 102 } 103 /* terminate connections of this link group abnormally */ 104 smc_lgr_terminate(smc_get_lgr(link)); 105 } 106 if (pnd_snd.handler) 107 pnd_snd.handler(&pnd_snd.priv, link, wc->status); 108 wake_up(&link->wr_tx_wait); 109 } 110 111 static void smc_wr_tx_tasklet_fn(unsigned long data) 112 { 113 struct smc_ib_device *dev = (struct smc_ib_device *)data; 114 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 115 int i = 0, rc; 116 int polled = 0; 117 118 again: 119 polled++; 120 do { 121 memset(&wc, 0, sizeof(wc)); 122 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); 123 if (polled == 1) { 124 ib_req_notify_cq(dev->roce_cq_send, 125 IB_CQ_NEXT_COMP | 126 IB_CQ_REPORT_MISSED_EVENTS); 127 } 128 if (!rc) 129 break; 130 for (i = 0; i < rc; i++) 131 smc_wr_tx_process_cqe(&wc[i]); 132 } while (rc > 0); 133 if (polled == 1) 134 goto again; 135 } 136 137 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 138 { 139 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 140 141 tasklet_schedule(&dev->send_tasklet); 142 } 143 144 /*---------------------------- request submission ---------------------------*/ 145 146 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) 147 { 148 *idx = link->wr_tx_cnt; 149 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { 150 if (!test_and_set_bit(*idx, link->wr_tx_mask)) 151 return 0; 152 } 153 *idx = link->wr_tx_cnt; 154 return -EBUSY; 155 } 156 157 /** 158 * smc_wr_tx_get_free_slot() - returns buffer for message assembly, 159 * and sets info for pending transmit tracking 160 * @link: Pointer to smc_link used to later send the message. 161 * @handler: Send completion handler function pointer. 162 * @wr_buf: Out value returns pointer to message buffer. 163 * @wr_pend_priv: Out value returns pointer serving as handler context. 164 * 165 * Return: 0 on success, or -errno on error. 166 */ 167 int smc_wr_tx_get_free_slot(struct smc_link *link, 168 smc_wr_tx_handler handler, 169 struct smc_wr_buf **wr_buf, 170 struct smc_wr_tx_pend_priv **wr_pend_priv) 171 { 172 struct smc_wr_tx_pend *wr_pend; 173 u32 idx = link->wr_tx_cnt; 174 struct ib_send_wr *wr_ib; 175 u64 wr_id; 176 int rc; 177 178 *wr_buf = NULL; 179 *wr_pend_priv = NULL; 180 if (in_softirq()) { 181 rc = smc_wr_tx_get_free_slot_index(link, &idx); 182 if (rc) 183 return rc; 184 } else { 185 struct smc_link_group *lgr; 186 187 lgr = smc_get_lgr(link); 188 rc = wait_event_timeout( 189 link->wr_tx_wait, 190 list_empty(&lgr->list) || /* lgr terminated */ 191 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), 192 SMC_WR_TX_WAIT_FREE_SLOT_TIME); 193 if (!rc) { 194 /* timeout - terminate connections */ 195 smc_lgr_terminate(lgr); 196 return -EPIPE; 197 } 198 if (idx == link->wr_tx_cnt) 199 return -EPIPE; 200 } 201 wr_id = smc_wr_tx_get_next_wr_id(link); 202 wr_pend = &link->wr_tx_pends[idx]; 203 wr_pend->wr_id = wr_id; 204 wr_pend->handler = handler; 205 wr_pend->link = link; 206 wr_pend->idx = idx; 207 wr_ib = &link->wr_tx_ibs[idx]; 208 wr_ib->wr_id = wr_id; 209 *wr_buf = &link->wr_tx_bufs[idx]; 210 *wr_pend_priv = &wr_pend->priv; 211 return 0; 212 } 213 214 int smc_wr_tx_put_slot(struct smc_link *link, 215 struct smc_wr_tx_pend_priv *wr_pend_priv) 216 { 217 struct smc_wr_tx_pend *pend; 218 219 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv); 220 if (pend->idx < link->wr_tx_cnt) { 221 /* clear the full struct smc_wr_tx_pend including .priv */ 222 memset(&link->wr_tx_pends[pend->idx], 0, 223 sizeof(link->wr_tx_pends[pend->idx])); 224 memset(&link->wr_tx_bufs[pend->idx], 0, 225 sizeof(link->wr_tx_bufs[pend->idx])); 226 test_and_clear_bit(pend->idx, link->wr_tx_mask); 227 return 1; 228 } 229 230 return 0; 231 } 232 233 /* Send prepared WR slot via ib_post_send. 234 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 235 */ 236 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) 237 { 238 struct ib_send_wr *failed_wr = NULL; 239 struct smc_wr_tx_pend *pend; 240 int rc; 241 242 ib_req_notify_cq(link->smcibdev->roce_cq_send, 243 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 244 pend = container_of(priv, struct smc_wr_tx_pend, priv); 245 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], 246 &failed_wr); 247 if (rc) { 248 smc_wr_tx_put_slot(link, priv); 249 smc_lgr_terminate(smc_get_lgr(link)); 250 } 251 return rc; 252 } 253 254 /* Register a memory region and wait for result. */ 255 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) 256 { 257 struct ib_send_wr *failed_wr = NULL; 258 int rc; 259 260 ib_req_notify_cq(link->smcibdev->roce_cq_send, 261 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 262 link->wr_reg_state = POSTED; 263 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; 264 link->wr_reg.mr = mr; 265 link->wr_reg.key = mr->rkey; 266 failed_wr = &link->wr_reg.wr; 267 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, &failed_wr); 268 WARN_ON(failed_wr != &link->wr_reg.wr); 269 if (rc) 270 return rc; 271 272 rc = wait_event_interruptible_timeout(link->wr_reg_wait, 273 (link->wr_reg_state != POSTED), 274 SMC_WR_REG_MR_WAIT_TIME); 275 if (!rc) { 276 /* timeout - terminate connections */ 277 smc_lgr_terminate(smc_get_lgr(link)); 278 return -EPIPE; 279 } 280 if (rc == -ERESTARTSYS) 281 return -EINTR; 282 switch (link->wr_reg_state) { 283 case CONFIRMED: 284 rc = 0; 285 break; 286 case FAILED: 287 rc = -EIO; 288 break; 289 case POSTED: 290 rc = -EPIPE; 291 break; 292 } 293 return rc; 294 } 295 296 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type, 297 smc_wr_tx_filter filter, 298 smc_wr_tx_dismisser dismisser, 299 unsigned long data) 300 { 301 struct smc_wr_tx_pend_priv *tx_pend; 302 struct smc_wr_rx_hdr *wr_tx; 303 int i; 304 305 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 306 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i]; 307 if (wr_tx->type != wr_tx_hdr_type) 308 continue; 309 tx_pend = &link->wr_tx_pends[i].priv; 310 if (filter(tx_pend, data)) 311 dismisser(tx_pend); 312 } 313 } 314 315 /****************************** receive queue ********************************/ 316 317 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler) 318 { 319 struct smc_wr_rx_handler *h_iter; 320 int rc = 0; 321 322 spin_lock(&smc_wr_rx_hash_lock); 323 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { 324 if (h_iter->type == handler->type) { 325 rc = -EEXIST; 326 goto out_unlock; 327 } 328 } 329 hash_add(smc_wr_rx_hash, &handler->list, handler->type); 330 out_unlock: 331 spin_unlock(&smc_wr_rx_hash_lock); 332 return rc; 333 } 334 335 /* Demultiplex a received work request based on the message type to its handler. 336 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs, 337 * and not being modified any more afterwards so we don't need to lock it. 338 */ 339 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc) 340 { 341 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 342 struct smc_wr_rx_handler *handler; 343 struct smc_wr_rx_hdr *wr_rx; 344 u64 temp_wr_id; 345 u32 index; 346 347 if (wc->byte_len < sizeof(*wr_rx)) 348 return; /* short message */ 349 temp_wr_id = wc->wr_id; 350 index = do_div(temp_wr_id, link->wr_rx_cnt); 351 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; 352 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { 353 if (handler->type == wr_rx->type) 354 handler->handler(wc, wr_rx); 355 } 356 } 357 358 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) 359 { 360 struct smc_link *link; 361 int i; 362 363 for (i = 0; i < num; i++) { 364 link = wc[i].qp->qp_context; 365 if (wc[i].status == IB_WC_SUCCESS) { 366 link->wr_rx_tstamp = jiffies; 367 smc_wr_rx_demultiplex(&wc[i]); 368 smc_wr_rx_post(link); /* refill WR RX */ 369 } else { 370 /* handle status errors */ 371 switch (wc[i].status) { 372 case IB_WC_RETRY_EXC_ERR: 373 case IB_WC_RNR_RETRY_EXC_ERR: 374 case IB_WC_WR_FLUSH_ERR: 375 /* terminate connections of this link group 376 * abnormally 377 */ 378 smc_lgr_terminate(smc_get_lgr(link)); 379 break; 380 default: 381 smc_wr_rx_post(link); /* refill WR RX */ 382 break; 383 } 384 } 385 } 386 } 387 388 static void smc_wr_rx_tasklet_fn(unsigned long data) 389 { 390 struct smc_ib_device *dev = (struct smc_ib_device *)data; 391 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 392 int polled = 0; 393 int rc; 394 395 again: 396 polled++; 397 do { 398 memset(&wc, 0, sizeof(wc)); 399 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); 400 if (polled == 1) { 401 ib_req_notify_cq(dev->roce_cq_recv, 402 IB_CQ_SOLICITED_MASK 403 | IB_CQ_REPORT_MISSED_EVENTS); 404 } 405 if (!rc) 406 break; 407 smc_wr_rx_process_cqes(&wc[0], rc); 408 } while (rc > 0); 409 if (polled == 1) 410 goto again; 411 } 412 413 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 414 { 415 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 416 417 tasklet_schedule(&dev->recv_tasklet); 418 } 419 420 int smc_wr_rx_post_init(struct smc_link *link) 421 { 422 u32 i; 423 int rc = 0; 424 425 for (i = 0; i < link->wr_rx_cnt; i++) 426 rc = smc_wr_rx_post(link); 427 return rc; 428 } 429 430 /***************************** init, exit, misc ******************************/ 431 432 void smc_wr_remember_qp_attr(struct smc_link *lnk) 433 { 434 struct ib_qp_attr *attr = &lnk->qp_attr; 435 struct ib_qp_init_attr init_attr; 436 437 memset(attr, 0, sizeof(*attr)); 438 memset(&init_attr, 0, sizeof(init_attr)); 439 ib_query_qp(lnk->roce_qp, attr, 440 IB_QP_STATE | 441 IB_QP_CUR_STATE | 442 IB_QP_PKEY_INDEX | 443 IB_QP_PORT | 444 IB_QP_QKEY | 445 IB_QP_AV | 446 IB_QP_PATH_MTU | 447 IB_QP_TIMEOUT | 448 IB_QP_RETRY_CNT | 449 IB_QP_RNR_RETRY | 450 IB_QP_RQ_PSN | 451 IB_QP_ALT_PATH | 452 IB_QP_MIN_RNR_TIMER | 453 IB_QP_SQ_PSN | 454 IB_QP_PATH_MIG_STATE | 455 IB_QP_CAP | 456 IB_QP_DEST_QPN, 457 &init_attr); 458 459 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, 460 lnk->qp_attr.cap.max_send_wr); 461 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, 462 lnk->qp_attr.cap.max_recv_wr); 463 } 464 465 static void smc_wr_init_sge(struct smc_link *lnk) 466 { 467 u32 i; 468 469 for (i = 0; i < lnk->wr_tx_cnt; i++) { 470 lnk->wr_tx_sges[i].addr = 471 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; 472 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; 473 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 474 lnk->wr_tx_ibs[i].next = NULL; 475 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; 476 lnk->wr_tx_ibs[i].num_sge = 1; 477 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; 478 lnk->wr_tx_ibs[i].send_flags = 479 IB_SEND_SIGNALED | IB_SEND_SOLICITED; 480 } 481 for (i = 0; i < lnk->wr_rx_cnt; i++) { 482 lnk->wr_rx_sges[i].addr = 483 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; 484 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE; 485 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 486 lnk->wr_rx_ibs[i].next = NULL; 487 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i]; 488 lnk->wr_rx_ibs[i].num_sge = 1; 489 } 490 lnk->wr_reg.wr.next = NULL; 491 lnk->wr_reg.wr.num_sge = 0; 492 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; 493 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; 494 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; 495 } 496 497 void smc_wr_free_link(struct smc_link *lnk) 498 { 499 struct ib_device *ibdev; 500 501 memset(lnk->wr_tx_mask, 0, 502 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 503 504 if (!lnk->smcibdev) 505 return; 506 ibdev = lnk->smcibdev->ibdev; 507 508 if (lnk->wr_rx_dma_addr) { 509 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 510 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 511 DMA_FROM_DEVICE); 512 lnk->wr_rx_dma_addr = 0; 513 } 514 if (lnk->wr_tx_dma_addr) { 515 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, 516 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 517 DMA_TO_DEVICE); 518 lnk->wr_tx_dma_addr = 0; 519 } 520 } 521 522 void smc_wr_free_link_mem(struct smc_link *lnk) 523 { 524 kfree(lnk->wr_tx_pends); 525 lnk->wr_tx_pends = NULL; 526 kfree(lnk->wr_tx_mask); 527 lnk->wr_tx_mask = NULL; 528 kfree(lnk->wr_tx_sges); 529 lnk->wr_tx_sges = NULL; 530 kfree(lnk->wr_rx_sges); 531 lnk->wr_rx_sges = NULL; 532 kfree(lnk->wr_rx_ibs); 533 lnk->wr_rx_ibs = NULL; 534 kfree(lnk->wr_tx_ibs); 535 lnk->wr_tx_ibs = NULL; 536 kfree(lnk->wr_tx_bufs); 537 lnk->wr_tx_bufs = NULL; 538 kfree(lnk->wr_rx_bufs); 539 lnk->wr_rx_bufs = NULL; 540 } 541 542 int smc_wr_alloc_link_mem(struct smc_link *link) 543 { 544 /* allocate link related memory */ 545 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); 546 if (!link->wr_tx_bufs) 547 goto no_mem; 548 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, 549 GFP_KERNEL); 550 if (!link->wr_rx_bufs) 551 goto no_mem_wr_tx_bufs; 552 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), 553 GFP_KERNEL); 554 if (!link->wr_tx_ibs) 555 goto no_mem_wr_rx_bufs; 556 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, 557 sizeof(link->wr_rx_ibs[0]), 558 GFP_KERNEL); 559 if (!link->wr_rx_ibs) 560 goto no_mem_wr_tx_ibs; 561 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), 562 GFP_KERNEL); 563 if (!link->wr_tx_sges) 564 goto no_mem_wr_rx_ibs; 565 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, 566 sizeof(link->wr_rx_sges[0]), 567 GFP_KERNEL); 568 if (!link->wr_rx_sges) 569 goto no_mem_wr_tx_sges; 570 link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT), 571 sizeof(*link->wr_tx_mask), 572 GFP_KERNEL); 573 if (!link->wr_tx_mask) 574 goto no_mem_wr_rx_sges; 575 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, 576 sizeof(link->wr_tx_pends[0]), 577 GFP_KERNEL); 578 if (!link->wr_tx_pends) 579 goto no_mem_wr_tx_mask; 580 return 0; 581 582 no_mem_wr_tx_mask: 583 kfree(link->wr_tx_mask); 584 no_mem_wr_rx_sges: 585 kfree(link->wr_rx_sges); 586 no_mem_wr_tx_sges: 587 kfree(link->wr_tx_sges); 588 no_mem_wr_rx_ibs: 589 kfree(link->wr_rx_ibs); 590 no_mem_wr_tx_ibs: 591 kfree(link->wr_tx_ibs); 592 no_mem_wr_rx_bufs: 593 kfree(link->wr_rx_bufs); 594 no_mem_wr_tx_bufs: 595 kfree(link->wr_tx_bufs); 596 no_mem: 597 return -ENOMEM; 598 } 599 600 void smc_wr_remove_dev(struct smc_ib_device *smcibdev) 601 { 602 tasklet_kill(&smcibdev->recv_tasklet); 603 tasklet_kill(&smcibdev->send_tasklet); 604 } 605 606 void smc_wr_add_dev(struct smc_ib_device *smcibdev) 607 { 608 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn, 609 (unsigned long)smcibdev); 610 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn, 611 (unsigned long)smcibdev); 612 } 613 614 int smc_wr_create_link(struct smc_link *lnk) 615 { 616 struct ib_device *ibdev = lnk->smcibdev->ibdev; 617 int rc = 0; 618 619 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); 620 lnk->wr_rx_id = 0; 621 lnk->wr_rx_dma_addr = ib_dma_map_single( 622 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 623 DMA_FROM_DEVICE); 624 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { 625 lnk->wr_rx_dma_addr = 0; 626 rc = -EIO; 627 goto out; 628 } 629 lnk->wr_tx_dma_addr = ib_dma_map_single( 630 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 631 DMA_TO_DEVICE); 632 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { 633 rc = -EIO; 634 goto dma_unmap; 635 } 636 smc_wr_init_sge(lnk); 637 memset(lnk->wr_tx_mask, 0, 638 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 639 init_waitqueue_head(&lnk->wr_tx_wait); 640 init_waitqueue_head(&lnk->wr_reg_wait); 641 return rc; 642 643 dma_unmap: 644 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 645 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 646 DMA_FROM_DEVICE); 647 lnk->wr_rx_dma_addr = 0; 648 out: 649 return rc; 650 } 651