1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Work Requests exploiting Infiniband API 6 * 7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively 8 * are submitted to either RC SQ or RC RQ respectively 9 * (reliably connected send/receive queue) 10 * and become work queue entries (WQEs). 11 * While an SQ WR/WQE is pending, we track it until transmission completion. 12 * Through a send or receive completion queue (CQ) respectively, 13 * we get completion queue entries (CQEs) [aka work completions (WCs)]. 14 * Since the CQ callback is called from IRQ context, we split work by using 15 * bottom halves implemented by tasklets. 16 * 17 * SMC uses this to exchange LLC (link layer control) 18 * and CDC (connection data control) messages. 19 * 20 * Copyright IBM Corp. 2016 21 * 22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> 23 */ 24 25 #include <linux/atomic.h> 26 #include <linux/hashtable.h> 27 #include <linux/wait.h> 28 #include <rdma/ib_verbs.h> 29 #include <asm/div64.h> 30 31 #include "smc.h" 32 #include "smc_wr.h" 33 34 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */ 35 36 #define SMC_WR_RX_HASH_BITS 4 37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS); 38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock); 39 40 struct smc_wr_tx_pend { /* control data for a pending send request */ 41 u64 wr_id; /* work request id sent */ 42 smc_wr_tx_handler handler; 43 enum ib_wc_status wc_status; /* CQE status */ 44 struct smc_link *link; 45 u32 idx; 46 struct smc_wr_tx_pend_priv priv; 47 u8 compl_requested; 48 }; 49 50 /******************************** send queue *********************************/ 51 52 /*------------------------------- completion --------------------------------*/ 53 54 /* returns true if at least one tx work request is pending on the given link */ 55 static inline bool smc_wr_is_tx_pend(struct smc_link *link) 56 { 57 if (find_first_bit(link->wr_tx_mask, link->wr_tx_cnt) != 58 link->wr_tx_cnt) { 59 return true; 60 } 61 return false; 62 } 63 64 /* wait till all pending tx work requests on the given link are completed */ 65 int smc_wr_tx_wait_no_pending_sends(struct smc_link *link) 66 { 67 if (wait_event_timeout(link->wr_tx_wait, !smc_wr_is_tx_pend(link), 68 SMC_WR_TX_WAIT_PENDING_TIME)) 69 return 0; 70 else /* timeout */ 71 return -EPIPE; 72 } 73 74 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) 75 { 76 u32 i; 77 78 for (i = 0; i < link->wr_tx_cnt; i++) { 79 if (link->wr_tx_pends[i].wr_id == wr_id) 80 return i; 81 } 82 return link->wr_tx_cnt; 83 } 84 85 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc) 86 { 87 struct smc_wr_tx_pend pnd_snd; 88 struct smc_link *link; 89 u32 pnd_snd_idx; 90 int i; 91 92 link = wc->qp->qp_context; 93 94 if (wc->opcode == IB_WC_REG_MR) { 95 if (wc->status) 96 link->wr_reg_state = FAILED; 97 else 98 link->wr_reg_state = CONFIRMED; 99 smc_wr_wakeup_reg_wait(link); 100 return; 101 } 102 103 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); 104 if (pnd_snd_idx == link->wr_tx_cnt) 105 return; 106 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; 107 if (link->wr_tx_pends[pnd_snd_idx].compl_requested) 108 complete(&link->wr_tx_compl[pnd_snd_idx]); 109 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd)); 110 /* clear the full struct smc_wr_tx_pend including .priv */ 111 memset(&link->wr_tx_pends[pnd_snd_idx], 0, 112 sizeof(link->wr_tx_pends[pnd_snd_idx])); 113 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, 114 sizeof(link->wr_tx_bufs[pnd_snd_idx])); 115 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) 116 return; 117 if (wc->status) { 118 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 119 /* clear full struct smc_wr_tx_pend including .priv */ 120 memset(&link->wr_tx_pends[i], 0, 121 sizeof(link->wr_tx_pends[i])); 122 memset(&link->wr_tx_bufs[i], 0, 123 sizeof(link->wr_tx_bufs[i])); 124 clear_bit(i, link->wr_tx_mask); 125 } 126 /* terminate link */ 127 smcr_link_down_cond_sched(link); 128 } 129 if (pnd_snd.handler) 130 pnd_snd.handler(&pnd_snd.priv, link, wc->status); 131 wake_up(&link->wr_tx_wait); 132 } 133 134 static void smc_wr_tx_tasklet_fn(struct tasklet_struct *t) 135 { 136 struct smc_ib_device *dev = from_tasklet(dev, t, send_tasklet); 137 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 138 int i = 0, rc; 139 int polled = 0; 140 141 again: 142 polled++; 143 do { 144 memset(&wc, 0, sizeof(wc)); 145 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); 146 if (polled == 1) { 147 ib_req_notify_cq(dev->roce_cq_send, 148 IB_CQ_NEXT_COMP | 149 IB_CQ_REPORT_MISSED_EVENTS); 150 } 151 if (!rc) 152 break; 153 for (i = 0; i < rc; i++) 154 smc_wr_tx_process_cqe(&wc[i]); 155 } while (rc > 0); 156 if (polled == 1) 157 goto again; 158 } 159 160 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 161 { 162 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 163 164 tasklet_schedule(&dev->send_tasklet); 165 } 166 167 /*---------------------------- request submission ---------------------------*/ 168 169 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) 170 { 171 *idx = link->wr_tx_cnt; 172 if (!smc_link_usable(link)) 173 return -ENOLINK; 174 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { 175 if (!test_and_set_bit(*idx, link->wr_tx_mask)) 176 return 0; 177 } 178 *idx = link->wr_tx_cnt; 179 return -EBUSY; 180 } 181 182 /** 183 * smc_wr_tx_get_free_slot() - returns buffer for message assembly, 184 * and sets info for pending transmit tracking 185 * @link: Pointer to smc_link used to later send the message. 186 * @handler: Send completion handler function pointer. 187 * @wr_buf: Out value returns pointer to message buffer. 188 * @wr_rdma_buf: Out value returns pointer to rdma work request. 189 * @wr_pend_priv: Out value returns pointer serving as handler context. 190 * 191 * Return: 0 on success, or -errno on error. 192 */ 193 int smc_wr_tx_get_free_slot(struct smc_link *link, 194 smc_wr_tx_handler handler, 195 struct smc_wr_buf **wr_buf, 196 struct smc_rdma_wr **wr_rdma_buf, 197 struct smc_wr_tx_pend_priv **wr_pend_priv) 198 { 199 struct smc_link_group *lgr = smc_get_lgr(link); 200 struct smc_wr_tx_pend *wr_pend; 201 u32 idx = link->wr_tx_cnt; 202 struct ib_send_wr *wr_ib; 203 u64 wr_id; 204 int rc; 205 206 *wr_buf = NULL; 207 *wr_pend_priv = NULL; 208 if (in_softirq() || lgr->terminating) { 209 rc = smc_wr_tx_get_free_slot_index(link, &idx); 210 if (rc) 211 return rc; 212 } else { 213 rc = wait_event_interruptible_timeout( 214 link->wr_tx_wait, 215 !smc_link_usable(link) || 216 lgr->terminating || 217 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), 218 SMC_WR_TX_WAIT_FREE_SLOT_TIME); 219 if (!rc) { 220 /* timeout - terminate link */ 221 smcr_link_down_cond_sched(link); 222 return -EPIPE; 223 } 224 if (idx == link->wr_tx_cnt) 225 return -EPIPE; 226 } 227 wr_id = smc_wr_tx_get_next_wr_id(link); 228 wr_pend = &link->wr_tx_pends[idx]; 229 wr_pend->wr_id = wr_id; 230 wr_pend->handler = handler; 231 wr_pend->link = link; 232 wr_pend->idx = idx; 233 wr_ib = &link->wr_tx_ibs[idx]; 234 wr_ib->wr_id = wr_id; 235 *wr_buf = &link->wr_tx_bufs[idx]; 236 if (wr_rdma_buf) 237 *wr_rdma_buf = &link->wr_tx_rdmas[idx]; 238 *wr_pend_priv = &wr_pend->priv; 239 return 0; 240 } 241 242 int smc_wr_tx_put_slot(struct smc_link *link, 243 struct smc_wr_tx_pend_priv *wr_pend_priv) 244 { 245 struct smc_wr_tx_pend *pend; 246 247 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv); 248 if (pend->idx < link->wr_tx_cnt) { 249 u32 idx = pend->idx; 250 251 /* clear the full struct smc_wr_tx_pend including .priv */ 252 memset(&link->wr_tx_pends[idx], 0, 253 sizeof(link->wr_tx_pends[idx])); 254 memset(&link->wr_tx_bufs[idx], 0, 255 sizeof(link->wr_tx_bufs[idx])); 256 test_and_clear_bit(idx, link->wr_tx_mask); 257 wake_up(&link->wr_tx_wait); 258 return 1; 259 } 260 261 return 0; 262 } 263 264 /* Send prepared WR slot via ib_post_send. 265 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 266 */ 267 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) 268 { 269 struct smc_wr_tx_pend *pend; 270 int rc; 271 272 ib_req_notify_cq(link->smcibdev->roce_cq_send, 273 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 274 pend = container_of(priv, struct smc_wr_tx_pend, priv); 275 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL); 276 if (rc) { 277 smc_wr_tx_put_slot(link, priv); 278 smcr_link_down_cond_sched(link); 279 } 280 return rc; 281 } 282 283 /* Send prepared WR slot via ib_post_send and wait for send completion 284 * notification. 285 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 286 */ 287 int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, 288 unsigned long timeout) 289 { 290 struct smc_wr_tx_pend *pend; 291 int rc; 292 293 pend = container_of(priv, struct smc_wr_tx_pend, priv); 294 pend->compl_requested = 1; 295 init_completion(&link->wr_tx_compl[pend->idx]); 296 297 rc = smc_wr_tx_send(link, priv); 298 if (rc) 299 return rc; 300 /* wait for completion by smc_wr_tx_process_cqe() */ 301 rc = wait_for_completion_interruptible_timeout( 302 &link->wr_tx_compl[pend->idx], timeout); 303 if (rc <= 0) 304 rc = -ENODATA; 305 if (rc > 0) 306 rc = 0; 307 return rc; 308 } 309 310 /* Register a memory region and wait for result. */ 311 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) 312 { 313 int rc; 314 315 ib_req_notify_cq(link->smcibdev->roce_cq_send, 316 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS); 317 link->wr_reg_state = POSTED; 318 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; 319 link->wr_reg.mr = mr; 320 link->wr_reg.key = mr->rkey; 321 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL); 322 if (rc) 323 return rc; 324 325 atomic_inc(&link->wr_reg_refcnt); 326 rc = wait_event_interruptible_timeout(link->wr_reg_wait, 327 (link->wr_reg_state != POSTED), 328 SMC_WR_REG_MR_WAIT_TIME); 329 if (atomic_dec_and_test(&link->wr_reg_refcnt)) 330 wake_up_all(&link->wr_reg_wait); 331 if (!rc) { 332 /* timeout - terminate link */ 333 smcr_link_down_cond_sched(link); 334 return -EPIPE; 335 } 336 if (rc == -ERESTARTSYS) 337 return -EINTR; 338 switch (link->wr_reg_state) { 339 case CONFIRMED: 340 rc = 0; 341 break; 342 case FAILED: 343 rc = -EIO; 344 break; 345 case POSTED: 346 rc = -EPIPE; 347 break; 348 } 349 return rc; 350 } 351 352 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type, 353 smc_wr_tx_filter filter, 354 smc_wr_tx_dismisser dismisser, 355 unsigned long data) 356 { 357 struct smc_wr_tx_pend_priv *tx_pend; 358 struct smc_wr_rx_hdr *wr_tx; 359 int i; 360 361 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 362 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i]; 363 if (wr_tx->type != wr_tx_hdr_type) 364 continue; 365 tx_pend = &link->wr_tx_pends[i].priv; 366 if (filter(tx_pend, data)) 367 dismisser(tx_pend); 368 } 369 } 370 371 /****************************** receive queue ********************************/ 372 373 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler) 374 { 375 struct smc_wr_rx_handler *h_iter; 376 int rc = 0; 377 378 spin_lock(&smc_wr_rx_hash_lock); 379 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { 380 if (h_iter->type == handler->type) { 381 rc = -EEXIST; 382 goto out_unlock; 383 } 384 } 385 hash_add(smc_wr_rx_hash, &handler->list, handler->type); 386 out_unlock: 387 spin_unlock(&smc_wr_rx_hash_lock); 388 return rc; 389 } 390 391 /* Demultiplex a received work request based on the message type to its handler. 392 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs, 393 * and not being modified any more afterwards so we don't need to lock it. 394 */ 395 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc) 396 { 397 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 398 struct smc_wr_rx_handler *handler; 399 struct smc_wr_rx_hdr *wr_rx; 400 u64 temp_wr_id; 401 u32 index; 402 403 if (wc->byte_len < sizeof(*wr_rx)) 404 return; /* short message */ 405 temp_wr_id = wc->wr_id; 406 index = do_div(temp_wr_id, link->wr_rx_cnt); 407 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; 408 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { 409 if (handler->type == wr_rx->type) 410 handler->handler(wc, wr_rx); 411 } 412 } 413 414 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) 415 { 416 struct smc_link *link; 417 int i; 418 419 for (i = 0; i < num; i++) { 420 link = wc[i].qp->qp_context; 421 if (wc[i].status == IB_WC_SUCCESS) { 422 link->wr_rx_tstamp = jiffies; 423 smc_wr_rx_demultiplex(&wc[i]); 424 smc_wr_rx_post(link); /* refill WR RX */ 425 } else { 426 /* handle status errors */ 427 switch (wc[i].status) { 428 case IB_WC_RETRY_EXC_ERR: 429 case IB_WC_RNR_RETRY_EXC_ERR: 430 case IB_WC_WR_FLUSH_ERR: 431 smcr_link_down_cond_sched(link); 432 break; 433 default: 434 smc_wr_rx_post(link); /* refill WR RX */ 435 break; 436 } 437 } 438 } 439 } 440 441 static void smc_wr_rx_tasklet_fn(struct tasklet_struct *t) 442 { 443 struct smc_ib_device *dev = from_tasklet(dev, t, recv_tasklet); 444 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 445 int polled = 0; 446 int rc; 447 448 again: 449 polled++; 450 do { 451 memset(&wc, 0, sizeof(wc)); 452 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); 453 if (polled == 1) { 454 ib_req_notify_cq(dev->roce_cq_recv, 455 IB_CQ_SOLICITED_MASK 456 | IB_CQ_REPORT_MISSED_EVENTS); 457 } 458 if (!rc) 459 break; 460 smc_wr_rx_process_cqes(&wc[0], rc); 461 } while (rc > 0); 462 if (polled == 1) 463 goto again; 464 } 465 466 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 467 { 468 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 469 470 tasklet_schedule(&dev->recv_tasklet); 471 } 472 473 int smc_wr_rx_post_init(struct smc_link *link) 474 { 475 u32 i; 476 int rc = 0; 477 478 for (i = 0; i < link->wr_rx_cnt; i++) 479 rc = smc_wr_rx_post(link); 480 return rc; 481 } 482 483 /***************************** init, exit, misc ******************************/ 484 485 void smc_wr_remember_qp_attr(struct smc_link *lnk) 486 { 487 struct ib_qp_attr *attr = &lnk->qp_attr; 488 struct ib_qp_init_attr init_attr; 489 490 memset(attr, 0, sizeof(*attr)); 491 memset(&init_attr, 0, sizeof(init_attr)); 492 ib_query_qp(lnk->roce_qp, attr, 493 IB_QP_STATE | 494 IB_QP_CUR_STATE | 495 IB_QP_PKEY_INDEX | 496 IB_QP_PORT | 497 IB_QP_QKEY | 498 IB_QP_AV | 499 IB_QP_PATH_MTU | 500 IB_QP_TIMEOUT | 501 IB_QP_RETRY_CNT | 502 IB_QP_RNR_RETRY | 503 IB_QP_RQ_PSN | 504 IB_QP_ALT_PATH | 505 IB_QP_MIN_RNR_TIMER | 506 IB_QP_SQ_PSN | 507 IB_QP_PATH_MIG_STATE | 508 IB_QP_CAP | 509 IB_QP_DEST_QPN, 510 &init_attr); 511 512 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, 513 lnk->qp_attr.cap.max_send_wr); 514 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, 515 lnk->qp_attr.cap.max_recv_wr); 516 } 517 518 static void smc_wr_init_sge(struct smc_link *lnk) 519 { 520 u32 i; 521 522 for (i = 0; i < lnk->wr_tx_cnt; i++) { 523 lnk->wr_tx_sges[i].addr = 524 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; 525 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; 526 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 527 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey = 528 lnk->roce_pd->local_dma_lkey; 529 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey = 530 lnk->roce_pd->local_dma_lkey; 531 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey = 532 lnk->roce_pd->local_dma_lkey; 533 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey = 534 lnk->roce_pd->local_dma_lkey; 535 lnk->wr_tx_ibs[i].next = NULL; 536 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; 537 lnk->wr_tx_ibs[i].num_sge = 1; 538 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; 539 lnk->wr_tx_ibs[i].send_flags = 540 IB_SEND_SIGNALED | IB_SEND_SOLICITED; 541 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE; 542 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE; 543 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list = 544 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge; 545 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list = 546 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge; 547 } 548 for (i = 0; i < lnk->wr_rx_cnt; i++) { 549 lnk->wr_rx_sges[i].addr = 550 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; 551 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE; 552 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 553 lnk->wr_rx_ibs[i].next = NULL; 554 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i]; 555 lnk->wr_rx_ibs[i].num_sge = 1; 556 } 557 lnk->wr_reg.wr.next = NULL; 558 lnk->wr_reg.wr.num_sge = 0; 559 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; 560 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; 561 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; 562 } 563 564 void smc_wr_free_link(struct smc_link *lnk) 565 { 566 struct ib_device *ibdev; 567 568 if (!lnk->smcibdev) 569 return; 570 ibdev = lnk->smcibdev->ibdev; 571 572 smc_wr_wakeup_reg_wait(lnk); 573 smc_wr_wakeup_tx_wait(lnk); 574 575 if (smc_wr_tx_wait_no_pending_sends(lnk)) 576 memset(lnk->wr_tx_mask, 0, 577 BITS_TO_LONGS(SMC_WR_BUF_CNT) * 578 sizeof(*lnk->wr_tx_mask)); 579 wait_event(lnk->wr_reg_wait, (!atomic_read(&lnk->wr_reg_refcnt))); 580 wait_event(lnk->wr_tx_wait, (!atomic_read(&lnk->wr_tx_refcnt))); 581 582 if (lnk->wr_rx_dma_addr) { 583 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 584 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 585 DMA_FROM_DEVICE); 586 lnk->wr_rx_dma_addr = 0; 587 } 588 if (lnk->wr_tx_dma_addr) { 589 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, 590 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 591 DMA_TO_DEVICE); 592 lnk->wr_tx_dma_addr = 0; 593 } 594 } 595 596 void smc_wr_free_link_mem(struct smc_link *lnk) 597 { 598 kfree(lnk->wr_tx_compl); 599 lnk->wr_tx_compl = NULL; 600 kfree(lnk->wr_tx_pends); 601 lnk->wr_tx_pends = NULL; 602 kfree(lnk->wr_tx_mask); 603 lnk->wr_tx_mask = NULL; 604 kfree(lnk->wr_tx_sges); 605 lnk->wr_tx_sges = NULL; 606 kfree(lnk->wr_tx_rdma_sges); 607 lnk->wr_tx_rdma_sges = NULL; 608 kfree(lnk->wr_rx_sges); 609 lnk->wr_rx_sges = NULL; 610 kfree(lnk->wr_tx_rdmas); 611 lnk->wr_tx_rdmas = NULL; 612 kfree(lnk->wr_rx_ibs); 613 lnk->wr_rx_ibs = NULL; 614 kfree(lnk->wr_tx_ibs); 615 lnk->wr_tx_ibs = NULL; 616 kfree(lnk->wr_tx_bufs); 617 lnk->wr_tx_bufs = NULL; 618 kfree(lnk->wr_rx_bufs); 619 lnk->wr_rx_bufs = NULL; 620 } 621 622 int smc_wr_alloc_link_mem(struct smc_link *link) 623 { 624 /* allocate link related memory */ 625 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); 626 if (!link->wr_tx_bufs) 627 goto no_mem; 628 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, 629 GFP_KERNEL); 630 if (!link->wr_rx_bufs) 631 goto no_mem_wr_tx_bufs; 632 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), 633 GFP_KERNEL); 634 if (!link->wr_tx_ibs) 635 goto no_mem_wr_rx_bufs; 636 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, 637 sizeof(link->wr_rx_ibs[0]), 638 GFP_KERNEL); 639 if (!link->wr_rx_ibs) 640 goto no_mem_wr_tx_ibs; 641 link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT, 642 sizeof(link->wr_tx_rdmas[0]), 643 GFP_KERNEL); 644 if (!link->wr_tx_rdmas) 645 goto no_mem_wr_rx_ibs; 646 link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT, 647 sizeof(link->wr_tx_rdma_sges[0]), 648 GFP_KERNEL); 649 if (!link->wr_tx_rdma_sges) 650 goto no_mem_wr_tx_rdmas; 651 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), 652 GFP_KERNEL); 653 if (!link->wr_tx_sges) 654 goto no_mem_wr_tx_rdma_sges; 655 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, 656 sizeof(link->wr_rx_sges[0]), 657 GFP_KERNEL); 658 if (!link->wr_rx_sges) 659 goto no_mem_wr_tx_sges; 660 link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT), 661 sizeof(*link->wr_tx_mask), 662 GFP_KERNEL); 663 if (!link->wr_tx_mask) 664 goto no_mem_wr_rx_sges; 665 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, 666 sizeof(link->wr_tx_pends[0]), 667 GFP_KERNEL); 668 if (!link->wr_tx_pends) 669 goto no_mem_wr_tx_mask; 670 link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT, 671 sizeof(link->wr_tx_compl[0]), 672 GFP_KERNEL); 673 if (!link->wr_tx_compl) 674 goto no_mem_wr_tx_pends; 675 return 0; 676 677 no_mem_wr_tx_pends: 678 kfree(link->wr_tx_pends); 679 no_mem_wr_tx_mask: 680 kfree(link->wr_tx_mask); 681 no_mem_wr_rx_sges: 682 kfree(link->wr_rx_sges); 683 no_mem_wr_tx_sges: 684 kfree(link->wr_tx_sges); 685 no_mem_wr_tx_rdma_sges: 686 kfree(link->wr_tx_rdma_sges); 687 no_mem_wr_tx_rdmas: 688 kfree(link->wr_tx_rdmas); 689 no_mem_wr_rx_ibs: 690 kfree(link->wr_rx_ibs); 691 no_mem_wr_tx_ibs: 692 kfree(link->wr_tx_ibs); 693 no_mem_wr_rx_bufs: 694 kfree(link->wr_rx_bufs); 695 no_mem_wr_tx_bufs: 696 kfree(link->wr_tx_bufs); 697 no_mem: 698 return -ENOMEM; 699 } 700 701 void smc_wr_remove_dev(struct smc_ib_device *smcibdev) 702 { 703 tasklet_kill(&smcibdev->recv_tasklet); 704 tasklet_kill(&smcibdev->send_tasklet); 705 } 706 707 void smc_wr_add_dev(struct smc_ib_device *smcibdev) 708 { 709 tasklet_setup(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn); 710 tasklet_setup(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn); 711 } 712 713 int smc_wr_create_link(struct smc_link *lnk) 714 { 715 struct ib_device *ibdev = lnk->smcibdev->ibdev; 716 int rc = 0; 717 718 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); 719 lnk->wr_rx_id = 0; 720 lnk->wr_rx_dma_addr = ib_dma_map_single( 721 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 722 DMA_FROM_DEVICE); 723 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { 724 lnk->wr_rx_dma_addr = 0; 725 rc = -EIO; 726 goto out; 727 } 728 lnk->wr_tx_dma_addr = ib_dma_map_single( 729 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 730 DMA_TO_DEVICE); 731 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { 732 rc = -EIO; 733 goto dma_unmap; 734 } 735 smc_wr_init_sge(lnk); 736 memset(lnk->wr_tx_mask, 0, 737 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 738 init_waitqueue_head(&lnk->wr_tx_wait); 739 atomic_set(&lnk->wr_tx_refcnt, 0); 740 init_waitqueue_head(&lnk->wr_reg_wait); 741 atomic_set(&lnk->wr_reg_refcnt, 0); 742 return rc; 743 744 dma_unmap: 745 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 746 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 747 DMA_FROM_DEVICE); 748 lnk->wr_rx_dma_addr = 0; 749 out: 750 return rc; 751 } 752