1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Work Requests exploiting Infiniband API 6 * 7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively 8 * are submitted to either RC SQ or RC RQ respectively 9 * (reliably connected send/receive queue) 10 * and become work queue entries (WQEs). 11 * While an SQ WR/WQE is pending, we track it until transmission completion. 12 * Through a send or receive completion queue (CQ) respectively, 13 * we get completion queue entries (CQEs) [aka work completions (WCs)]. 14 * Since the CQ callback is called from IRQ context, we split work by using 15 * bottom halves implemented by tasklets. 16 * 17 * SMC uses this to exchange LLC (link layer control) 18 * and CDC (connection data control) messages. 19 * 20 * Copyright IBM Corp. 2016 21 * 22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> 23 */ 24 25 #include <linux/atomic.h> 26 #include <linux/hashtable.h> 27 #include <linux/wait.h> 28 #include <rdma/ib_verbs.h> 29 #include <asm/div64.h> 30 31 #include "smc.h" 32 #include "smc_wr.h" 33 34 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */ 35 36 #define SMC_WR_RX_HASH_BITS 4 37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS); 38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock); 39 40 struct smc_wr_tx_pend { /* control data for a pending send request */ 41 u64 wr_id; /* work request id sent */ 42 smc_wr_tx_handler handler; 43 enum ib_wc_status wc_status; /* CQE status */ 44 struct smc_link *link; 45 u32 idx; 46 struct smc_wr_tx_pend_priv priv; 47 }; 48 49 /******************************** send queue *********************************/ 50 51 /*------------------------------- completion --------------------------------*/ 52 53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) 54 { 55 u32 i; 56 57 for (i = 0; i < link->wr_tx_cnt; i++) { 58 if (link->wr_tx_pends[i].wr_id == wr_id) 59 return i; 60 } 61 return link->wr_tx_cnt; 62 } 63 64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc) 65 { 66 struct smc_wr_tx_pend pnd_snd; 67 struct smc_link *link; 68 u32 pnd_snd_idx; 69 int i; 70 71 link = wc->qp->qp_context; 72 73 if (wc->opcode == IB_WC_REG_MR) { 74 if (wc->status) 75 link->wr_reg_state = FAILED; 76 else 77 link->wr_reg_state = CONFIRMED; 78 wake_up(&link->wr_reg_wait); 79 return; 80 } 81 82 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); 83 if (pnd_snd_idx == link->wr_tx_cnt) 84 return; 85 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; 86 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd)); 87 /* clear the full struct smc_wr_tx_pend including .priv */ 88 memset(&link->wr_tx_pends[pnd_snd_idx], 0, 89 sizeof(link->wr_tx_pends[pnd_snd_idx])); 90 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, 91 sizeof(link->wr_tx_bufs[pnd_snd_idx])); 92 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) 93 return; 94 if (wc->status) { 95 struct smc_link_group *lgr; 96 97 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 98 /* clear full struct smc_wr_tx_pend including .priv */ 99 memset(&link->wr_tx_pends[i], 0, 100 sizeof(link->wr_tx_pends[i])); 101 memset(&link->wr_tx_bufs[i], 0, 102 sizeof(link->wr_tx_bufs[i])); 103 clear_bit(i, link->wr_tx_mask); 104 } 105 /* terminate connections of this link group abnormally */ 106 lgr = container_of(link, struct smc_link_group, 107 lnk[SMC_SINGLE_LINK]); 108 smc_lgr_terminate(lgr); 109 } 110 if (pnd_snd.handler) 111 pnd_snd.handler(&pnd_snd.priv, link, wc->status); 112 wake_up(&link->wr_tx_wait); 113 } 114 115 static void smc_wr_tx_tasklet_fn(unsigned long data) 116 { 117 struct smc_ib_device *dev = (struct smc_ib_device *)data; 118 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 119 int i = 0, rc; 120 int polled = 0; 121 122 again: 123 polled++; 124 do { 125 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); 126 if (polled == 1) { 127 ib_req_notify_cq(dev->roce_cq_send, 128 IB_CQ_NEXT_COMP | 129 IB_CQ_REPORT_MISSED_EVENTS); 130 } 131 if (!rc) 132 break; 133 for (i = 0; i < rc; i++) 134 smc_wr_tx_process_cqe(&wc[i]); 135 } while (rc > 0); 136 if (polled == 1) 137 goto again; 138 } 139 140 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 141 { 142 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 143 144 tasklet_schedule(&dev->send_tasklet); 145 } 146 147 /*---------------------------- request submission ---------------------------*/ 148 149 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) 150 { 151 *idx = link->wr_tx_cnt; 152 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { 153 if (!test_and_set_bit(*idx, link->wr_tx_mask)) 154 return 0; 155 } 156 *idx = link->wr_tx_cnt; 157 return -EBUSY; 158 } 159 160 /** 161 * smc_wr_tx_get_free_slot() - returns buffer for message assembly, 162 * and sets info for pending transmit tracking 163 * @link: Pointer to smc_link used to later send the message. 164 * @handler: Send completion handler function pointer. 165 * @wr_buf: Out value returns pointer to message buffer. 166 * @wr_pend_priv: Out value returns pointer serving as handler context. 167 * 168 * Return: 0 on success, or -errno on error. 169 */ 170 int smc_wr_tx_get_free_slot(struct smc_link *link, 171 smc_wr_tx_handler handler, 172 struct smc_wr_buf **wr_buf, 173 struct smc_wr_tx_pend_priv **wr_pend_priv) 174 { 175 struct smc_wr_tx_pend *wr_pend; 176 struct ib_send_wr *wr_ib; 177 u64 wr_id; 178 u32 idx; 179 int rc; 180 181 *wr_buf = NULL; 182 *wr_pend_priv = NULL; 183 if (in_softirq()) { 184 rc = smc_wr_tx_get_free_slot_index(link, &idx); 185 if (rc) 186 return rc; 187 } else { 188 rc = wait_event_interruptible_timeout( 189 link->wr_tx_wait, 190 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), 191 SMC_WR_TX_WAIT_FREE_SLOT_TIME); 192 if (!rc) { 193 /* timeout - terminate connections */ 194 struct smc_link_group *lgr; 195 196 lgr = container_of(link, struct smc_link_group, 197 lnk[SMC_SINGLE_LINK]); 198 smc_lgr_terminate(lgr); 199 return -EPIPE; 200 } 201 if (rc == -ERESTARTSYS) 202 return -EINTR; 203 if (idx == link->wr_tx_cnt) 204 return -EPIPE; 205 } 206 wr_id = smc_wr_tx_get_next_wr_id(link); 207 wr_pend = &link->wr_tx_pends[idx]; 208 wr_pend->wr_id = wr_id; 209 wr_pend->handler = handler; 210 wr_pend->link = link; 211 wr_pend->idx = idx; 212 wr_ib = &link->wr_tx_ibs[idx]; 213 wr_ib->wr_id = wr_id; 214 *wr_buf = &link->wr_tx_bufs[idx]; 215 *wr_pend_priv = &wr_pend->priv; 216 return 0; 217 } 218 219 int smc_wr_tx_put_slot(struct smc_link *link, 220 struct smc_wr_tx_pend_priv *wr_pend_priv) 221 { 222 struct smc_wr_tx_pend *pend; 223 224 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv); 225 if (pend->idx < link->wr_tx_cnt) { 226 /* clear the full struct smc_wr_tx_pend including .priv */ 227 memset(&link->wr_tx_pends[pend->idx], 0, 228 sizeof(link->wr_tx_pends[pend->idx])); 229 memset(&link->wr_tx_bufs[pend->idx], 0, 230 sizeof(link->wr_tx_bufs[pend->idx])); 231 test_and_clear_bit(pend->idx, link->wr_tx_mask); 232 return 1; 233 } 234 235 return 0; 236 } 237 238 /* Send prepared WR slot via ib_post_send. 239 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 240 */ 241 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) 242 { 243 struct ib_send_wr *failed_wr = NULL; 244 struct smc_wr_tx_pend *pend; 245 int rc; 246 247 ib_req_notify_cq(link->smcibdev->roce_cq_send, 248 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 249 pend = container_of(priv, struct smc_wr_tx_pend, priv); 250 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], 251 &failed_wr); 252 if (rc) 253 smc_wr_tx_put_slot(link, priv); 254 return rc; 255 } 256 257 /* Register a memory region and wait for result. */ 258 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) 259 { 260 struct ib_send_wr *failed_wr = NULL; 261 int rc; 262 263 ib_req_notify_cq(link->smcibdev->roce_cq_send, 264 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 265 link->wr_reg_state = POSTED; 266 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; 267 link->wr_reg.mr = mr; 268 link->wr_reg.key = mr->rkey; 269 failed_wr = &link->wr_reg.wr; 270 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, &failed_wr); 271 WARN_ON(failed_wr != &link->wr_reg.wr); 272 if (rc) 273 return rc; 274 275 rc = wait_event_interruptible_timeout(link->wr_reg_wait, 276 (link->wr_reg_state != POSTED), 277 SMC_WR_REG_MR_WAIT_TIME); 278 if (!rc) { 279 /* timeout - terminate connections */ 280 struct smc_link_group *lgr; 281 282 lgr = container_of(link, struct smc_link_group, 283 lnk[SMC_SINGLE_LINK]); 284 smc_lgr_terminate(lgr); 285 return -EPIPE; 286 } 287 if (rc == -ERESTARTSYS) 288 return -EINTR; 289 switch (link->wr_reg_state) { 290 case CONFIRMED: 291 rc = 0; 292 break; 293 case FAILED: 294 rc = -EIO; 295 break; 296 case POSTED: 297 rc = -EPIPE; 298 break; 299 } 300 return rc; 301 } 302 303 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_rx_hdr_type, 304 smc_wr_tx_filter filter, 305 smc_wr_tx_dismisser dismisser, 306 unsigned long data) 307 { 308 struct smc_wr_tx_pend_priv *tx_pend; 309 struct smc_wr_rx_hdr *wr_rx; 310 int i; 311 312 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 313 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i]; 314 if (wr_rx->type != wr_rx_hdr_type) 315 continue; 316 tx_pend = &link->wr_tx_pends[i].priv; 317 if (filter(tx_pend, data)) 318 dismisser(tx_pend); 319 } 320 } 321 322 bool smc_wr_tx_has_pending(struct smc_link *link, u8 wr_rx_hdr_type, 323 smc_wr_tx_filter filter, unsigned long data) 324 { 325 struct smc_wr_tx_pend_priv *tx_pend; 326 struct smc_wr_rx_hdr *wr_rx; 327 int i; 328 329 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 330 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i]; 331 if (wr_rx->type != wr_rx_hdr_type) 332 continue; 333 tx_pend = &link->wr_tx_pends[i].priv; 334 if (filter(tx_pend, data)) 335 return true; 336 } 337 return false; 338 } 339 340 /****************************** receive queue ********************************/ 341 342 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler) 343 { 344 struct smc_wr_rx_handler *h_iter; 345 int rc = 0; 346 347 spin_lock(&smc_wr_rx_hash_lock); 348 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { 349 if (h_iter->type == handler->type) { 350 rc = -EEXIST; 351 goto out_unlock; 352 } 353 } 354 hash_add(smc_wr_rx_hash, &handler->list, handler->type); 355 out_unlock: 356 spin_unlock(&smc_wr_rx_hash_lock); 357 return rc; 358 } 359 360 /* Demultiplex a received work request based on the message type to its handler. 361 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs, 362 * and not being modified any more afterwards so we don't need to lock it. 363 */ 364 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc) 365 { 366 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 367 struct smc_wr_rx_handler *handler; 368 struct smc_wr_rx_hdr *wr_rx; 369 u64 temp_wr_id; 370 u32 index; 371 372 if (wc->byte_len < sizeof(*wr_rx)) 373 return; /* short message */ 374 temp_wr_id = wc->wr_id; 375 index = do_div(temp_wr_id, link->wr_rx_cnt); 376 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; 377 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { 378 if (handler->type == wr_rx->type) 379 handler->handler(wc, wr_rx); 380 } 381 } 382 383 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) 384 { 385 struct smc_link *link; 386 int i; 387 388 for (i = 0; i < num; i++) { 389 link = wc[i].qp->qp_context; 390 if (wc[i].status == IB_WC_SUCCESS) { 391 smc_wr_rx_demultiplex(&wc[i]); 392 smc_wr_rx_post(link); /* refill WR RX */ 393 } else { 394 struct smc_link_group *lgr; 395 396 /* handle status errors */ 397 switch (wc[i].status) { 398 case IB_WC_RETRY_EXC_ERR: 399 case IB_WC_RNR_RETRY_EXC_ERR: 400 case IB_WC_WR_FLUSH_ERR: 401 /* terminate connections of this link group 402 * abnormally 403 */ 404 lgr = container_of(link, struct smc_link_group, 405 lnk[SMC_SINGLE_LINK]); 406 smc_lgr_terminate(lgr); 407 break; 408 default: 409 smc_wr_rx_post(link); /* refill WR RX */ 410 break; 411 } 412 } 413 } 414 } 415 416 static void smc_wr_rx_tasklet_fn(unsigned long data) 417 { 418 struct smc_ib_device *dev = (struct smc_ib_device *)data; 419 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 420 int polled = 0; 421 int rc; 422 423 again: 424 polled++; 425 do { 426 memset(&wc, 0, sizeof(wc)); 427 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); 428 if (polled == 1) { 429 ib_req_notify_cq(dev->roce_cq_recv, 430 IB_CQ_SOLICITED_MASK 431 | IB_CQ_REPORT_MISSED_EVENTS); 432 } 433 if (!rc) 434 break; 435 smc_wr_rx_process_cqes(&wc[0], rc); 436 } while (rc > 0); 437 if (polled == 1) 438 goto again; 439 } 440 441 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 442 { 443 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 444 445 tasklet_schedule(&dev->recv_tasklet); 446 } 447 448 int smc_wr_rx_post_init(struct smc_link *link) 449 { 450 u32 i; 451 int rc = 0; 452 453 for (i = 0; i < link->wr_rx_cnt; i++) 454 rc = smc_wr_rx_post(link); 455 return rc; 456 } 457 458 /***************************** init, exit, misc ******************************/ 459 460 void smc_wr_remember_qp_attr(struct smc_link *lnk) 461 { 462 struct ib_qp_attr *attr = &lnk->qp_attr; 463 struct ib_qp_init_attr init_attr; 464 465 memset(attr, 0, sizeof(*attr)); 466 memset(&init_attr, 0, sizeof(init_attr)); 467 ib_query_qp(lnk->roce_qp, attr, 468 IB_QP_STATE | 469 IB_QP_CUR_STATE | 470 IB_QP_PKEY_INDEX | 471 IB_QP_PORT | 472 IB_QP_QKEY | 473 IB_QP_AV | 474 IB_QP_PATH_MTU | 475 IB_QP_TIMEOUT | 476 IB_QP_RETRY_CNT | 477 IB_QP_RNR_RETRY | 478 IB_QP_RQ_PSN | 479 IB_QP_ALT_PATH | 480 IB_QP_MIN_RNR_TIMER | 481 IB_QP_SQ_PSN | 482 IB_QP_PATH_MIG_STATE | 483 IB_QP_CAP | 484 IB_QP_DEST_QPN, 485 &init_attr); 486 487 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, 488 lnk->qp_attr.cap.max_send_wr); 489 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, 490 lnk->qp_attr.cap.max_recv_wr); 491 } 492 493 static void smc_wr_init_sge(struct smc_link *lnk) 494 { 495 u32 i; 496 497 for (i = 0; i < lnk->wr_tx_cnt; i++) { 498 lnk->wr_tx_sges[i].addr = 499 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; 500 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; 501 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 502 lnk->wr_tx_ibs[i].next = NULL; 503 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; 504 lnk->wr_tx_ibs[i].num_sge = 1; 505 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; 506 lnk->wr_tx_ibs[i].send_flags = 507 IB_SEND_SIGNALED | IB_SEND_SOLICITED; 508 } 509 for (i = 0; i < lnk->wr_rx_cnt; i++) { 510 lnk->wr_rx_sges[i].addr = 511 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; 512 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE; 513 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 514 lnk->wr_rx_ibs[i].next = NULL; 515 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i]; 516 lnk->wr_rx_ibs[i].num_sge = 1; 517 } 518 lnk->wr_reg.wr.next = NULL; 519 lnk->wr_reg.wr.num_sge = 0; 520 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; 521 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; 522 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; 523 } 524 525 void smc_wr_free_link(struct smc_link *lnk) 526 { 527 struct ib_device *ibdev; 528 529 memset(lnk->wr_tx_mask, 0, 530 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 531 532 if (!lnk->smcibdev) 533 return; 534 ibdev = lnk->smcibdev->ibdev; 535 536 if (lnk->wr_rx_dma_addr) { 537 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 538 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 539 DMA_FROM_DEVICE); 540 lnk->wr_rx_dma_addr = 0; 541 } 542 if (lnk->wr_tx_dma_addr) { 543 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, 544 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 545 DMA_TO_DEVICE); 546 lnk->wr_tx_dma_addr = 0; 547 } 548 } 549 550 void smc_wr_free_link_mem(struct smc_link *lnk) 551 { 552 kfree(lnk->wr_tx_pends); 553 lnk->wr_tx_pends = NULL; 554 kfree(lnk->wr_tx_mask); 555 lnk->wr_tx_mask = NULL; 556 kfree(lnk->wr_tx_sges); 557 lnk->wr_tx_sges = NULL; 558 kfree(lnk->wr_rx_sges); 559 lnk->wr_rx_sges = NULL; 560 kfree(lnk->wr_rx_ibs); 561 lnk->wr_rx_ibs = NULL; 562 kfree(lnk->wr_tx_ibs); 563 lnk->wr_tx_ibs = NULL; 564 kfree(lnk->wr_tx_bufs); 565 lnk->wr_tx_bufs = NULL; 566 kfree(lnk->wr_rx_bufs); 567 lnk->wr_rx_bufs = NULL; 568 } 569 570 int smc_wr_alloc_link_mem(struct smc_link *link) 571 { 572 /* allocate link related memory */ 573 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); 574 if (!link->wr_tx_bufs) 575 goto no_mem; 576 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, 577 GFP_KERNEL); 578 if (!link->wr_rx_bufs) 579 goto no_mem_wr_tx_bufs; 580 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), 581 GFP_KERNEL); 582 if (!link->wr_tx_ibs) 583 goto no_mem_wr_rx_bufs; 584 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, 585 sizeof(link->wr_rx_ibs[0]), 586 GFP_KERNEL); 587 if (!link->wr_rx_ibs) 588 goto no_mem_wr_tx_ibs; 589 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), 590 GFP_KERNEL); 591 if (!link->wr_tx_sges) 592 goto no_mem_wr_rx_ibs; 593 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, 594 sizeof(link->wr_rx_sges[0]), 595 GFP_KERNEL); 596 if (!link->wr_rx_sges) 597 goto no_mem_wr_tx_sges; 598 link->wr_tx_mask = kzalloc( 599 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*link->wr_tx_mask), 600 GFP_KERNEL); 601 if (!link->wr_tx_mask) 602 goto no_mem_wr_rx_sges; 603 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, 604 sizeof(link->wr_tx_pends[0]), 605 GFP_KERNEL); 606 if (!link->wr_tx_pends) 607 goto no_mem_wr_tx_mask; 608 return 0; 609 610 no_mem_wr_tx_mask: 611 kfree(link->wr_tx_mask); 612 no_mem_wr_rx_sges: 613 kfree(link->wr_rx_sges); 614 no_mem_wr_tx_sges: 615 kfree(link->wr_tx_sges); 616 no_mem_wr_rx_ibs: 617 kfree(link->wr_rx_ibs); 618 no_mem_wr_tx_ibs: 619 kfree(link->wr_tx_ibs); 620 no_mem_wr_rx_bufs: 621 kfree(link->wr_rx_bufs); 622 no_mem_wr_tx_bufs: 623 kfree(link->wr_tx_bufs); 624 no_mem: 625 return -ENOMEM; 626 } 627 628 void smc_wr_remove_dev(struct smc_ib_device *smcibdev) 629 { 630 tasklet_kill(&smcibdev->recv_tasklet); 631 tasklet_kill(&smcibdev->send_tasklet); 632 } 633 634 void smc_wr_add_dev(struct smc_ib_device *smcibdev) 635 { 636 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn, 637 (unsigned long)smcibdev); 638 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn, 639 (unsigned long)smcibdev); 640 } 641 642 int smc_wr_create_link(struct smc_link *lnk) 643 { 644 struct ib_device *ibdev = lnk->smcibdev->ibdev; 645 int rc = 0; 646 647 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); 648 lnk->wr_rx_id = 0; 649 lnk->wr_rx_dma_addr = ib_dma_map_single( 650 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 651 DMA_FROM_DEVICE); 652 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { 653 lnk->wr_rx_dma_addr = 0; 654 rc = -EIO; 655 goto out; 656 } 657 lnk->wr_tx_dma_addr = ib_dma_map_single( 658 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 659 DMA_TO_DEVICE); 660 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { 661 rc = -EIO; 662 goto dma_unmap; 663 } 664 smc_wr_init_sge(lnk); 665 memset(lnk->wr_tx_mask, 0, 666 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 667 init_waitqueue_head(&lnk->wr_tx_wait); 668 init_waitqueue_head(&lnk->wr_reg_wait); 669 return rc; 670 671 dma_unmap: 672 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 673 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 674 DMA_FROM_DEVICE); 675 lnk->wr_rx_dma_addr = 0; 676 out: 677 return rc; 678 } 679