1 /* 2 * Shared Memory Communications over RDMA (SMC-R) and RoCE 3 * 4 * Work Requests exploiting Infiniband API 5 * 6 * Work requests (WR) of type ib_post_send or ib_post_recv respectively 7 * are submitted to either RC SQ or RC RQ respectively 8 * (reliably connected send/receive queue) 9 * and become work queue entries (WQEs). 10 * While an SQ WR/WQE is pending, we track it until transmission completion. 11 * Through a send or receive completion queue (CQ) respectively, 12 * we get completion queue entries (CQEs) [aka work completions (WCs)]. 13 * Since the CQ callback is called from IRQ context, we split work by using 14 * bottom halves implemented by tasklets. 15 * 16 * SMC uses this to exchange LLC (link layer control) 17 * and CDC (connection data control) messages. 18 * 19 * Copyright IBM Corp. 2016 20 * 21 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> 22 */ 23 24 #include <linux/atomic.h> 25 #include <linux/hashtable.h> 26 #include <linux/wait.h> 27 #include <rdma/ib_verbs.h> 28 #include <asm/div64.h> 29 30 #include "smc.h" 31 #include "smc_wr.h" 32 33 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */ 34 35 #define SMC_WR_RX_HASH_BITS 4 36 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS); 37 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock); 38 39 struct smc_wr_tx_pend { /* control data for a pending send request */ 40 u64 wr_id; /* work request id sent */ 41 smc_wr_tx_handler handler; 42 enum ib_wc_status wc_status; /* CQE status */ 43 struct smc_link *link; 44 u32 idx; 45 struct smc_wr_tx_pend_priv priv; 46 }; 47 48 /******************************** send queue *********************************/ 49 50 /*------------------------------- completion --------------------------------*/ 51 52 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) 53 { 54 u32 i; 55 56 for (i = 0; i < link->wr_tx_cnt; i++) { 57 if (link->wr_tx_pends[i].wr_id == wr_id) 58 return i; 59 } 60 return link->wr_tx_cnt; 61 } 62 63 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc) 64 { 65 struct smc_wr_tx_pend pnd_snd; 66 struct smc_link *link; 67 u32 pnd_snd_idx; 68 int i; 69 70 link = wc->qp->qp_context; 71 72 if (wc->opcode == IB_WC_REG_MR) { 73 if (wc->status) 74 link->wr_reg_state = FAILED; 75 else 76 link->wr_reg_state = CONFIRMED; 77 wake_up(&link->wr_reg_wait); 78 return; 79 } 80 81 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); 82 if (pnd_snd_idx == link->wr_tx_cnt) 83 return; 84 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; 85 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd)); 86 /* clear the full struct smc_wr_tx_pend including .priv */ 87 memset(&link->wr_tx_pends[pnd_snd_idx], 0, 88 sizeof(link->wr_tx_pends[pnd_snd_idx])); 89 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, 90 sizeof(link->wr_tx_bufs[pnd_snd_idx])); 91 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) 92 return; 93 if (wc->status) { 94 struct smc_link_group *lgr; 95 96 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 97 /* clear full struct smc_wr_tx_pend including .priv */ 98 memset(&link->wr_tx_pends[i], 0, 99 sizeof(link->wr_tx_pends[i])); 100 memset(&link->wr_tx_bufs[i], 0, 101 sizeof(link->wr_tx_bufs[i])); 102 clear_bit(i, link->wr_tx_mask); 103 } 104 /* terminate connections of this link group abnormally */ 105 lgr = container_of(link, struct smc_link_group, 106 lnk[SMC_SINGLE_LINK]); 107 smc_lgr_terminate(lgr); 108 } 109 if (pnd_snd.handler) 110 pnd_snd.handler(&pnd_snd.priv, link, wc->status); 111 wake_up(&link->wr_tx_wait); 112 } 113 114 static void smc_wr_tx_tasklet_fn(unsigned long data) 115 { 116 struct smc_ib_device *dev = (struct smc_ib_device *)data; 117 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 118 int i = 0, rc; 119 int polled = 0; 120 121 again: 122 polled++; 123 do { 124 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); 125 if (polled == 1) { 126 ib_req_notify_cq(dev->roce_cq_send, 127 IB_CQ_NEXT_COMP | 128 IB_CQ_REPORT_MISSED_EVENTS); 129 } 130 if (!rc) 131 break; 132 for (i = 0; i < rc; i++) 133 smc_wr_tx_process_cqe(&wc[i]); 134 } while (rc > 0); 135 if (polled == 1) 136 goto again; 137 } 138 139 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 140 { 141 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 142 143 tasklet_schedule(&dev->send_tasklet); 144 } 145 146 /*---------------------------- request submission ---------------------------*/ 147 148 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) 149 { 150 *idx = link->wr_tx_cnt; 151 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { 152 if (!test_and_set_bit(*idx, link->wr_tx_mask)) 153 return 0; 154 } 155 *idx = link->wr_tx_cnt; 156 return -EBUSY; 157 } 158 159 /** 160 * smc_wr_tx_get_free_slot() - returns buffer for message assembly, 161 * and sets info for pending transmit tracking 162 * @link: Pointer to smc_link used to later send the message. 163 * @handler: Send completion handler function pointer. 164 * @wr_buf: Out value returns pointer to message buffer. 165 * @wr_pend_priv: Out value returns pointer serving as handler context. 166 * 167 * Return: 0 on success, or -errno on error. 168 */ 169 int smc_wr_tx_get_free_slot(struct smc_link *link, 170 smc_wr_tx_handler handler, 171 struct smc_wr_buf **wr_buf, 172 struct smc_wr_tx_pend_priv **wr_pend_priv) 173 { 174 struct smc_wr_tx_pend *wr_pend; 175 struct ib_send_wr *wr_ib; 176 u64 wr_id; 177 u32 idx; 178 int rc; 179 180 *wr_buf = NULL; 181 *wr_pend_priv = NULL; 182 if (in_softirq()) { 183 rc = smc_wr_tx_get_free_slot_index(link, &idx); 184 if (rc) 185 return rc; 186 } else { 187 rc = wait_event_interruptible_timeout( 188 link->wr_tx_wait, 189 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), 190 SMC_WR_TX_WAIT_FREE_SLOT_TIME); 191 if (!rc) { 192 /* timeout - terminate connections */ 193 struct smc_link_group *lgr; 194 195 lgr = container_of(link, struct smc_link_group, 196 lnk[SMC_SINGLE_LINK]); 197 smc_lgr_terminate(lgr); 198 return -EPIPE; 199 } 200 if (rc == -ERESTARTSYS) 201 return -EINTR; 202 if (idx == link->wr_tx_cnt) 203 return -EPIPE; 204 } 205 wr_id = smc_wr_tx_get_next_wr_id(link); 206 wr_pend = &link->wr_tx_pends[idx]; 207 wr_pend->wr_id = wr_id; 208 wr_pend->handler = handler; 209 wr_pend->link = link; 210 wr_pend->idx = idx; 211 wr_ib = &link->wr_tx_ibs[idx]; 212 wr_ib->wr_id = wr_id; 213 *wr_buf = &link->wr_tx_bufs[idx]; 214 *wr_pend_priv = &wr_pend->priv; 215 return 0; 216 } 217 218 int smc_wr_tx_put_slot(struct smc_link *link, 219 struct smc_wr_tx_pend_priv *wr_pend_priv) 220 { 221 struct smc_wr_tx_pend *pend; 222 223 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv); 224 if (pend->idx < link->wr_tx_cnt) { 225 /* clear the full struct smc_wr_tx_pend including .priv */ 226 memset(&link->wr_tx_pends[pend->idx], 0, 227 sizeof(link->wr_tx_pends[pend->idx])); 228 memset(&link->wr_tx_bufs[pend->idx], 0, 229 sizeof(link->wr_tx_bufs[pend->idx])); 230 test_and_clear_bit(pend->idx, link->wr_tx_mask); 231 return 1; 232 } 233 234 return 0; 235 } 236 237 /* Send prepared WR slot via ib_post_send. 238 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 239 */ 240 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) 241 { 242 struct ib_send_wr *failed_wr = NULL; 243 struct smc_wr_tx_pend *pend; 244 int rc; 245 246 ib_req_notify_cq(link->smcibdev->roce_cq_send, 247 IB_CQ_SOLICITED_MASK | IB_CQ_REPORT_MISSED_EVENTS); 248 pend = container_of(priv, struct smc_wr_tx_pend, priv); 249 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], 250 &failed_wr); 251 if (rc) 252 smc_wr_tx_put_slot(link, priv); 253 return rc; 254 } 255 256 /* Register a memory region and wait for result. */ 257 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) 258 { 259 struct ib_send_wr *failed_wr = NULL; 260 int rc; 261 262 ib_req_notify_cq(link->smcibdev->roce_cq_send, 263 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 264 link->wr_reg_state = POSTED; 265 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; 266 link->wr_reg.mr = mr; 267 link->wr_reg.key = mr->rkey; 268 failed_wr = &link->wr_reg.wr; 269 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, &failed_wr); 270 WARN_ON(failed_wr != &link->wr_reg.wr); 271 if (rc) 272 return rc; 273 274 rc = wait_event_interruptible_timeout(link->wr_reg_wait, 275 (link->wr_reg_state != POSTED), 276 SMC_WR_REG_MR_WAIT_TIME); 277 if (!rc) { 278 /* timeout - terminate connections */ 279 struct smc_link_group *lgr; 280 281 lgr = container_of(link, struct smc_link_group, 282 lnk[SMC_SINGLE_LINK]); 283 smc_lgr_terminate(lgr); 284 return -EPIPE; 285 } 286 if (rc == -ERESTARTSYS) 287 return -EINTR; 288 switch (link->wr_reg_state) { 289 case CONFIRMED: 290 rc = 0; 291 break; 292 case FAILED: 293 rc = -EIO; 294 break; 295 case POSTED: 296 rc = -EPIPE; 297 break; 298 } 299 return rc; 300 } 301 302 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_rx_hdr_type, 303 smc_wr_tx_filter filter, 304 smc_wr_tx_dismisser dismisser, 305 unsigned long data) 306 { 307 struct smc_wr_tx_pend_priv *tx_pend; 308 struct smc_wr_rx_hdr *wr_rx; 309 int i; 310 311 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 312 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i]; 313 if (wr_rx->type != wr_rx_hdr_type) 314 continue; 315 tx_pend = &link->wr_tx_pends[i].priv; 316 if (filter(tx_pend, data)) 317 dismisser(tx_pend); 318 } 319 } 320 321 bool smc_wr_tx_has_pending(struct smc_link *link, u8 wr_rx_hdr_type, 322 smc_wr_tx_filter filter, unsigned long data) 323 { 324 struct smc_wr_tx_pend_priv *tx_pend; 325 struct smc_wr_rx_hdr *wr_rx; 326 int i; 327 328 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 329 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i]; 330 if (wr_rx->type != wr_rx_hdr_type) 331 continue; 332 tx_pend = &link->wr_tx_pends[i].priv; 333 if (filter(tx_pend, data)) 334 return true; 335 } 336 return false; 337 } 338 339 /****************************** receive queue ********************************/ 340 341 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler) 342 { 343 struct smc_wr_rx_handler *h_iter; 344 int rc = 0; 345 346 spin_lock(&smc_wr_rx_hash_lock); 347 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { 348 if (h_iter->type == handler->type) { 349 rc = -EEXIST; 350 goto out_unlock; 351 } 352 } 353 hash_add(smc_wr_rx_hash, &handler->list, handler->type); 354 out_unlock: 355 spin_unlock(&smc_wr_rx_hash_lock); 356 return rc; 357 } 358 359 /* Demultiplex a received work request based on the message type to its handler. 360 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs, 361 * and not being modified any more afterwards so we don't need to lock it. 362 */ 363 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc) 364 { 365 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 366 struct smc_wr_rx_handler *handler; 367 struct smc_wr_rx_hdr *wr_rx; 368 u64 temp_wr_id; 369 u32 index; 370 371 if (wc->byte_len < sizeof(*wr_rx)) 372 return; /* short message */ 373 temp_wr_id = wc->wr_id; 374 index = do_div(temp_wr_id, link->wr_rx_cnt); 375 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; 376 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { 377 if (handler->type == wr_rx->type) 378 handler->handler(wc, wr_rx); 379 } 380 } 381 382 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) 383 { 384 struct smc_link *link; 385 int i; 386 387 for (i = 0; i < num; i++) { 388 link = wc[i].qp->qp_context; 389 if (wc[i].status == IB_WC_SUCCESS) { 390 smc_wr_rx_demultiplex(&wc[i]); 391 smc_wr_rx_post(link); /* refill WR RX */ 392 } else { 393 struct smc_link_group *lgr; 394 395 /* handle status errors */ 396 switch (wc[i].status) { 397 case IB_WC_RETRY_EXC_ERR: 398 case IB_WC_RNR_RETRY_EXC_ERR: 399 case IB_WC_WR_FLUSH_ERR: 400 /* terminate connections of this link group 401 * abnormally 402 */ 403 lgr = container_of(link, struct smc_link_group, 404 lnk[SMC_SINGLE_LINK]); 405 smc_lgr_terminate(lgr); 406 break; 407 default: 408 smc_wr_rx_post(link); /* refill WR RX */ 409 break; 410 } 411 } 412 } 413 } 414 415 static void smc_wr_rx_tasklet_fn(unsigned long data) 416 { 417 struct smc_ib_device *dev = (struct smc_ib_device *)data; 418 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 419 int polled = 0; 420 int rc; 421 422 again: 423 polled++; 424 do { 425 memset(&wc, 0, sizeof(wc)); 426 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); 427 if (polled == 1) { 428 ib_req_notify_cq(dev->roce_cq_recv, 429 IB_CQ_SOLICITED_MASK 430 | IB_CQ_REPORT_MISSED_EVENTS); 431 } 432 if (!rc) 433 break; 434 smc_wr_rx_process_cqes(&wc[0], rc); 435 } while (rc > 0); 436 if (polled == 1) 437 goto again; 438 } 439 440 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 441 { 442 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 443 444 tasklet_schedule(&dev->recv_tasklet); 445 } 446 447 int smc_wr_rx_post_init(struct smc_link *link) 448 { 449 u32 i; 450 int rc = 0; 451 452 for (i = 0; i < link->wr_rx_cnt; i++) 453 rc = smc_wr_rx_post(link); 454 return rc; 455 } 456 457 /***************************** init, exit, misc ******************************/ 458 459 void smc_wr_remember_qp_attr(struct smc_link *lnk) 460 { 461 struct ib_qp_attr *attr = &lnk->qp_attr; 462 struct ib_qp_init_attr init_attr; 463 464 memset(attr, 0, sizeof(*attr)); 465 memset(&init_attr, 0, sizeof(init_attr)); 466 ib_query_qp(lnk->roce_qp, attr, 467 IB_QP_STATE | 468 IB_QP_CUR_STATE | 469 IB_QP_PKEY_INDEX | 470 IB_QP_PORT | 471 IB_QP_QKEY | 472 IB_QP_AV | 473 IB_QP_PATH_MTU | 474 IB_QP_TIMEOUT | 475 IB_QP_RETRY_CNT | 476 IB_QP_RNR_RETRY | 477 IB_QP_RQ_PSN | 478 IB_QP_ALT_PATH | 479 IB_QP_MIN_RNR_TIMER | 480 IB_QP_SQ_PSN | 481 IB_QP_PATH_MIG_STATE | 482 IB_QP_CAP | 483 IB_QP_DEST_QPN, 484 &init_attr); 485 486 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, 487 lnk->qp_attr.cap.max_send_wr); 488 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, 489 lnk->qp_attr.cap.max_recv_wr); 490 } 491 492 static void smc_wr_init_sge(struct smc_link *lnk) 493 { 494 u32 i; 495 496 for (i = 0; i < lnk->wr_tx_cnt; i++) { 497 lnk->wr_tx_sges[i].addr = 498 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; 499 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; 500 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 501 lnk->wr_tx_ibs[i].next = NULL; 502 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; 503 lnk->wr_tx_ibs[i].num_sge = 1; 504 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; 505 lnk->wr_tx_ibs[i].send_flags = 506 IB_SEND_SIGNALED | IB_SEND_SOLICITED; 507 } 508 for (i = 0; i < lnk->wr_rx_cnt; i++) { 509 lnk->wr_rx_sges[i].addr = 510 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; 511 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE; 512 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 513 lnk->wr_rx_ibs[i].next = NULL; 514 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i]; 515 lnk->wr_rx_ibs[i].num_sge = 1; 516 } 517 lnk->wr_reg.wr.next = NULL; 518 lnk->wr_reg.wr.num_sge = 0; 519 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; 520 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; 521 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; 522 } 523 524 void smc_wr_free_link(struct smc_link *lnk) 525 { 526 struct ib_device *ibdev; 527 528 memset(lnk->wr_tx_mask, 0, 529 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 530 531 if (!lnk->smcibdev) 532 return; 533 ibdev = lnk->smcibdev->ibdev; 534 535 if (lnk->wr_rx_dma_addr) { 536 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 537 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 538 DMA_FROM_DEVICE); 539 lnk->wr_rx_dma_addr = 0; 540 } 541 if (lnk->wr_tx_dma_addr) { 542 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, 543 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 544 DMA_TO_DEVICE); 545 lnk->wr_tx_dma_addr = 0; 546 } 547 } 548 549 void smc_wr_free_link_mem(struct smc_link *lnk) 550 { 551 kfree(lnk->wr_tx_pends); 552 lnk->wr_tx_pends = NULL; 553 kfree(lnk->wr_tx_mask); 554 lnk->wr_tx_mask = NULL; 555 kfree(lnk->wr_tx_sges); 556 lnk->wr_tx_sges = NULL; 557 kfree(lnk->wr_rx_sges); 558 lnk->wr_rx_sges = NULL; 559 kfree(lnk->wr_rx_ibs); 560 lnk->wr_rx_ibs = NULL; 561 kfree(lnk->wr_tx_ibs); 562 lnk->wr_tx_ibs = NULL; 563 kfree(lnk->wr_tx_bufs); 564 lnk->wr_tx_bufs = NULL; 565 kfree(lnk->wr_rx_bufs); 566 lnk->wr_rx_bufs = NULL; 567 } 568 569 int smc_wr_alloc_link_mem(struct smc_link *link) 570 { 571 /* allocate link related memory */ 572 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); 573 if (!link->wr_tx_bufs) 574 goto no_mem; 575 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, 576 GFP_KERNEL); 577 if (!link->wr_rx_bufs) 578 goto no_mem_wr_tx_bufs; 579 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), 580 GFP_KERNEL); 581 if (!link->wr_tx_ibs) 582 goto no_mem_wr_rx_bufs; 583 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, 584 sizeof(link->wr_rx_ibs[0]), 585 GFP_KERNEL); 586 if (!link->wr_rx_ibs) 587 goto no_mem_wr_tx_ibs; 588 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), 589 GFP_KERNEL); 590 if (!link->wr_tx_sges) 591 goto no_mem_wr_rx_ibs; 592 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, 593 sizeof(link->wr_rx_sges[0]), 594 GFP_KERNEL); 595 if (!link->wr_rx_sges) 596 goto no_mem_wr_tx_sges; 597 link->wr_tx_mask = kzalloc( 598 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*link->wr_tx_mask), 599 GFP_KERNEL); 600 if (!link->wr_tx_mask) 601 goto no_mem_wr_rx_sges; 602 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, 603 sizeof(link->wr_tx_pends[0]), 604 GFP_KERNEL); 605 if (!link->wr_tx_pends) 606 goto no_mem_wr_tx_mask; 607 return 0; 608 609 no_mem_wr_tx_mask: 610 kfree(link->wr_tx_mask); 611 no_mem_wr_rx_sges: 612 kfree(link->wr_rx_sges); 613 no_mem_wr_tx_sges: 614 kfree(link->wr_tx_sges); 615 no_mem_wr_rx_ibs: 616 kfree(link->wr_rx_ibs); 617 no_mem_wr_tx_ibs: 618 kfree(link->wr_tx_ibs); 619 no_mem_wr_rx_bufs: 620 kfree(link->wr_rx_bufs); 621 no_mem_wr_tx_bufs: 622 kfree(link->wr_tx_bufs); 623 no_mem: 624 return -ENOMEM; 625 } 626 627 void smc_wr_remove_dev(struct smc_ib_device *smcibdev) 628 { 629 tasklet_kill(&smcibdev->recv_tasklet); 630 tasklet_kill(&smcibdev->send_tasklet); 631 } 632 633 void smc_wr_add_dev(struct smc_ib_device *smcibdev) 634 { 635 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn, 636 (unsigned long)smcibdev); 637 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn, 638 (unsigned long)smcibdev); 639 } 640 641 int smc_wr_create_link(struct smc_link *lnk) 642 { 643 struct ib_device *ibdev = lnk->smcibdev->ibdev; 644 int rc = 0; 645 646 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); 647 lnk->wr_rx_id = 0; 648 lnk->wr_rx_dma_addr = ib_dma_map_single( 649 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 650 DMA_FROM_DEVICE); 651 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { 652 lnk->wr_rx_dma_addr = 0; 653 rc = -EIO; 654 goto out; 655 } 656 lnk->wr_tx_dma_addr = ib_dma_map_single( 657 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 658 DMA_TO_DEVICE); 659 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { 660 rc = -EIO; 661 goto dma_unmap; 662 } 663 smc_wr_init_sge(lnk); 664 memset(lnk->wr_tx_mask, 0, 665 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 666 init_waitqueue_head(&lnk->wr_tx_wait); 667 init_waitqueue_head(&lnk->wr_reg_wait); 668 return rc; 669 670 dma_unmap: 671 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 672 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 673 DMA_FROM_DEVICE); 674 lnk->wr_rx_dma_addr = 0; 675 out: 676 return rc; 677 } 678