1 /* 2 * Shared Memory Communications over RDMA (SMC-R) and RoCE 3 * 4 * Work Requests exploiting Infiniband API 5 * 6 * Work requests (WR) of type ib_post_send or ib_post_recv respectively 7 * are submitted to either RC SQ or RC RQ respectively 8 * (reliably connected send/receive queue) 9 * and become work queue entries (WQEs). 10 * While an SQ WR/WQE is pending, we track it until transmission completion. 11 * Through a send or receive completion queue (CQ) respectively, 12 * we get completion queue entries (CQEs) [aka work completions (WCs)]. 13 * Since the CQ callback is called from IRQ context, we split work by using 14 * bottom halves implemented by tasklets. 15 * 16 * SMC uses this to exchange LLC (link layer control) 17 * and CDC (connection data control) messages. 18 * 19 * Copyright IBM Corp. 2016 20 * 21 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com> 22 */ 23 24 #include <linux/atomic.h> 25 #include <linux/hashtable.h> 26 #include <linux/wait.h> 27 #include <rdma/ib_verbs.h> 28 #include <asm/div64.h> 29 30 #include "smc.h" 31 #include "smc_wr.h" 32 33 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */ 34 35 #define SMC_WR_RX_HASH_BITS 4 36 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS); 37 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock); 38 39 struct smc_wr_tx_pend { /* control data for a pending send request */ 40 u64 wr_id; /* work request id sent */ 41 smc_wr_tx_handler handler; 42 enum ib_wc_status wc_status; /* CQE status */ 43 struct smc_link *link; 44 u32 idx; 45 struct smc_wr_tx_pend_priv priv; 46 }; 47 48 /******************************** send queue *********************************/ 49 50 /*------------------------------- completion --------------------------------*/ 51 52 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) 53 { 54 u32 i; 55 56 for (i = 0; i < link->wr_tx_cnt; i++) { 57 if (link->wr_tx_pends[i].wr_id == wr_id) 58 return i; 59 } 60 return link->wr_tx_cnt; 61 } 62 63 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc) 64 { 65 struct smc_wr_tx_pend pnd_snd; 66 struct smc_link *link; 67 u32 pnd_snd_idx; 68 int i; 69 70 link = wc->qp->qp_context; 71 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); 72 if (pnd_snd_idx == link->wr_tx_cnt) 73 return; 74 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; 75 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd)); 76 /* clear the full struct smc_wr_tx_pend including .priv */ 77 memset(&link->wr_tx_pends[pnd_snd_idx], 0, 78 sizeof(link->wr_tx_pends[pnd_snd_idx])); 79 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, 80 sizeof(link->wr_tx_bufs[pnd_snd_idx])); 81 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) 82 return; 83 if (wc->status) { 84 struct smc_link_group *lgr; 85 86 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 87 /* clear full struct smc_wr_tx_pend including .priv */ 88 memset(&link->wr_tx_pends[i], 0, 89 sizeof(link->wr_tx_pends[i])); 90 memset(&link->wr_tx_bufs[i], 0, 91 sizeof(link->wr_tx_bufs[i])); 92 clear_bit(i, link->wr_tx_mask); 93 } 94 /* terminate connections of this link group abnormally */ 95 lgr = container_of(link, struct smc_link_group, 96 lnk[SMC_SINGLE_LINK]); 97 smc_lgr_terminate(lgr); 98 } 99 if (pnd_snd.handler) 100 pnd_snd.handler(&pnd_snd.priv, link, wc->status); 101 wake_up(&link->wr_tx_wait); 102 } 103 104 static void smc_wr_tx_tasklet_fn(unsigned long data) 105 { 106 struct smc_ib_device *dev = (struct smc_ib_device *)data; 107 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 108 int i = 0, rc; 109 int polled = 0; 110 111 again: 112 polled++; 113 do { 114 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc); 115 if (polled == 1) { 116 ib_req_notify_cq(dev->roce_cq_send, 117 IB_CQ_NEXT_COMP | 118 IB_CQ_REPORT_MISSED_EVENTS); 119 } 120 if (!rc) 121 break; 122 for (i = 0; i < rc; i++) 123 smc_wr_tx_process_cqe(&wc[i]); 124 } while (rc > 0); 125 if (polled == 1) 126 goto again; 127 } 128 129 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 130 { 131 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 132 133 tasklet_schedule(&dev->send_tasklet); 134 } 135 136 /*---------------------------- request submission ---------------------------*/ 137 138 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) 139 { 140 *idx = link->wr_tx_cnt; 141 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { 142 if (!test_and_set_bit(*idx, link->wr_tx_mask)) 143 return 0; 144 } 145 *idx = link->wr_tx_cnt; 146 return -EBUSY; 147 } 148 149 /** 150 * smc_wr_tx_get_free_slot() - returns buffer for message assembly, 151 * and sets info for pending transmit tracking 152 * @link: Pointer to smc_link used to later send the message. 153 * @handler: Send completion handler function pointer. 154 * @wr_buf: Out value returns pointer to message buffer. 155 * @wr_pend_priv: Out value returns pointer serving as handler context. 156 * 157 * Return: 0 on success, or -errno on error. 158 */ 159 int smc_wr_tx_get_free_slot(struct smc_link *link, 160 smc_wr_tx_handler handler, 161 struct smc_wr_buf **wr_buf, 162 struct smc_wr_tx_pend_priv **wr_pend_priv) 163 { 164 struct smc_wr_tx_pend *wr_pend; 165 struct ib_send_wr *wr_ib; 166 u64 wr_id; 167 u32 idx; 168 int rc; 169 170 *wr_buf = NULL; 171 *wr_pend_priv = NULL; 172 if (in_softirq()) { 173 rc = smc_wr_tx_get_free_slot_index(link, &idx); 174 if (rc) 175 return rc; 176 } else { 177 rc = wait_event_interruptible_timeout( 178 link->wr_tx_wait, 179 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), 180 SMC_WR_TX_WAIT_FREE_SLOT_TIME); 181 if (!rc) { 182 /* timeout - terminate connections */ 183 struct smc_link_group *lgr; 184 185 lgr = container_of(link, struct smc_link_group, 186 lnk[SMC_SINGLE_LINK]); 187 smc_lgr_terminate(lgr); 188 return -EPIPE; 189 } 190 if (rc == -ERESTARTSYS) 191 return -EINTR; 192 if (idx == link->wr_tx_cnt) 193 return -EPIPE; 194 } 195 wr_id = smc_wr_tx_get_next_wr_id(link); 196 wr_pend = &link->wr_tx_pends[idx]; 197 wr_pend->wr_id = wr_id; 198 wr_pend->handler = handler; 199 wr_pend->link = link; 200 wr_pend->idx = idx; 201 wr_ib = &link->wr_tx_ibs[idx]; 202 wr_ib->wr_id = wr_id; 203 *wr_buf = &link->wr_tx_bufs[idx]; 204 *wr_pend_priv = &wr_pend->priv; 205 return 0; 206 } 207 208 int smc_wr_tx_put_slot(struct smc_link *link, 209 struct smc_wr_tx_pend_priv *wr_pend_priv) 210 { 211 struct smc_wr_tx_pend *pend; 212 213 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv); 214 if (pend->idx < link->wr_tx_cnt) { 215 /* clear the full struct smc_wr_tx_pend including .priv */ 216 memset(&link->wr_tx_pends[pend->idx], 0, 217 sizeof(link->wr_tx_pends[pend->idx])); 218 memset(&link->wr_tx_bufs[pend->idx], 0, 219 sizeof(link->wr_tx_bufs[pend->idx])); 220 test_and_clear_bit(pend->idx, link->wr_tx_mask); 221 return 1; 222 } 223 224 return 0; 225 } 226 227 /* Send prepared WR slot via ib_post_send. 228 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer 229 */ 230 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) 231 { 232 struct ib_send_wr *failed_wr = NULL; 233 struct smc_wr_tx_pend *pend; 234 int rc; 235 236 ib_req_notify_cq(link->smcibdev->roce_cq_send, 237 IB_CQ_SOLICITED_MASK | IB_CQ_REPORT_MISSED_EVENTS); 238 pend = container_of(priv, struct smc_wr_tx_pend, priv); 239 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], 240 &failed_wr); 241 if (rc) 242 smc_wr_tx_put_slot(link, priv); 243 return rc; 244 } 245 246 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_rx_hdr_type, 247 smc_wr_tx_filter filter, 248 smc_wr_tx_dismisser dismisser, 249 unsigned long data) 250 { 251 struct smc_wr_tx_pend_priv *tx_pend; 252 struct smc_wr_rx_hdr *wr_rx; 253 int i; 254 255 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 256 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i]; 257 if (wr_rx->type != wr_rx_hdr_type) 258 continue; 259 tx_pend = &link->wr_tx_pends[i].priv; 260 if (filter(tx_pend, data)) 261 dismisser(tx_pend); 262 } 263 } 264 265 bool smc_wr_tx_has_pending(struct smc_link *link, u8 wr_rx_hdr_type, 266 smc_wr_tx_filter filter, unsigned long data) 267 { 268 struct smc_wr_tx_pend_priv *tx_pend; 269 struct smc_wr_rx_hdr *wr_rx; 270 int i; 271 272 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) { 273 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[i]; 274 if (wr_rx->type != wr_rx_hdr_type) 275 continue; 276 tx_pend = &link->wr_tx_pends[i].priv; 277 if (filter(tx_pend, data)) 278 return true; 279 } 280 return false; 281 } 282 283 /****************************** receive queue ********************************/ 284 285 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler) 286 { 287 struct smc_wr_rx_handler *h_iter; 288 int rc = 0; 289 290 spin_lock(&smc_wr_rx_hash_lock); 291 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) { 292 if (h_iter->type == handler->type) { 293 rc = -EEXIST; 294 goto out_unlock; 295 } 296 } 297 hash_add(smc_wr_rx_hash, &handler->list, handler->type); 298 out_unlock: 299 spin_unlock(&smc_wr_rx_hash_lock); 300 return rc; 301 } 302 303 /* Demultiplex a received work request based on the message type to its handler. 304 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs, 305 * and not being modified any more afterwards so we don't need to lock it. 306 */ 307 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc) 308 { 309 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 310 struct smc_wr_rx_handler *handler; 311 struct smc_wr_rx_hdr *wr_rx; 312 u64 temp_wr_id; 313 u32 index; 314 315 if (wc->byte_len < sizeof(*wr_rx)) 316 return; /* short message */ 317 temp_wr_id = wc->wr_id; 318 index = do_div(temp_wr_id, link->wr_rx_cnt); 319 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; 320 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) { 321 if (handler->type == wr_rx->type) 322 handler->handler(wc, wr_rx); 323 } 324 } 325 326 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) 327 { 328 struct smc_link *link; 329 int i; 330 331 for (i = 0; i < num; i++) { 332 link = wc[i].qp->qp_context; 333 if (wc[i].status == IB_WC_SUCCESS) { 334 smc_wr_rx_demultiplex(&wc[i]); 335 smc_wr_rx_post(link); /* refill WR RX */ 336 } else { 337 struct smc_link_group *lgr; 338 339 /* handle status errors */ 340 switch (wc[i].status) { 341 case IB_WC_RETRY_EXC_ERR: 342 case IB_WC_RNR_RETRY_EXC_ERR: 343 case IB_WC_WR_FLUSH_ERR: 344 /* terminate connections of this link group 345 * abnormally 346 */ 347 lgr = container_of(link, struct smc_link_group, 348 lnk[SMC_SINGLE_LINK]); 349 smc_lgr_terminate(lgr); 350 break; 351 default: 352 smc_wr_rx_post(link); /* refill WR RX */ 353 break; 354 } 355 } 356 } 357 } 358 359 static void smc_wr_rx_tasklet_fn(unsigned long data) 360 { 361 struct smc_ib_device *dev = (struct smc_ib_device *)data; 362 struct ib_wc wc[SMC_WR_MAX_POLL_CQE]; 363 int polled = 0; 364 int rc; 365 366 again: 367 polled++; 368 do { 369 memset(&wc, 0, sizeof(wc)); 370 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc); 371 if (polled == 1) { 372 ib_req_notify_cq(dev->roce_cq_recv, 373 IB_CQ_SOLICITED_MASK 374 | IB_CQ_REPORT_MISSED_EVENTS); 375 } 376 if (!rc) 377 break; 378 smc_wr_rx_process_cqes(&wc[0], rc); 379 } while (rc > 0); 380 if (polled == 1) 381 goto again; 382 } 383 384 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context) 385 { 386 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context; 387 388 tasklet_schedule(&dev->recv_tasklet); 389 } 390 391 int smc_wr_rx_post_init(struct smc_link *link) 392 { 393 u32 i; 394 int rc = 0; 395 396 for (i = 0; i < link->wr_rx_cnt; i++) 397 rc = smc_wr_rx_post(link); 398 return rc; 399 } 400 401 /***************************** init, exit, misc ******************************/ 402 403 void smc_wr_remember_qp_attr(struct smc_link *lnk) 404 { 405 struct ib_qp_attr *attr = &lnk->qp_attr; 406 struct ib_qp_init_attr init_attr; 407 408 memset(attr, 0, sizeof(*attr)); 409 memset(&init_attr, 0, sizeof(init_attr)); 410 ib_query_qp(lnk->roce_qp, attr, 411 IB_QP_STATE | 412 IB_QP_CUR_STATE | 413 IB_QP_PKEY_INDEX | 414 IB_QP_PORT | 415 IB_QP_QKEY | 416 IB_QP_AV | 417 IB_QP_PATH_MTU | 418 IB_QP_TIMEOUT | 419 IB_QP_RETRY_CNT | 420 IB_QP_RNR_RETRY | 421 IB_QP_RQ_PSN | 422 IB_QP_ALT_PATH | 423 IB_QP_MIN_RNR_TIMER | 424 IB_QP_SQ_PSN | 425 IB_QP_PATH_MIG_STATE | 426 IB_QP_CAP | 427 IB_QP_DEST_QPN, 428 &init_attr); 429 430 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, 431 lnk->qp_attr.cap.max_send_wr); 432 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, 433 lnk->qp_attr.cap.max_recv_wr); 434 } 435 436 static void smc_wr_init_sge(struct smc_link *lnk) 437 { 438 u32 i; 439 440 for (i = 0; i < lnk->wr_tx_cnt; i++) { 441 lnk->wr_tx_sges[i].addr = 442 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; 443 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; 444 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 445 lnk->wr_tx_ibs[i].next = NULL; 446 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; 447 lnk->wr_tx_ibs[i].num_sge = 1; 448 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; 449 lnk->wr_tx_ibs[i].send_flags = 450 IB_SEND_SIGNALED | IB_SEND_SOLICITED; 451 } 452 for (i = 0; i < lnk->wr_rx_cnt; i++) { 453 lnk->wr_rx_sges[i].addr = 454 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; 455 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE; 456 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; 457 lnk->wr_rx_ibs[i].next = NULL; 458 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i]; 459 lnk->wr_rx_ibs[i].num_sge = 1; 460 } 461 } 462 463 void smc_wr_free_link(struct smc_link *lnk) 464 { 465 struct ib_device *ibdev; 466 467 memset(lnk->wr_tx_mask, 0, 468 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 469 470 if (!lnk->smcibdev) 471 return; 472 ibdev = lnk->smcibdev->ibdev; 473 474 if (lnk->wr_rx_dma_addr) { 475 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 476 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 477 DMA_FROM_DEVICE); 478 lnk->wr_rx_dma_addr = 0; 479 } 480 if (lnk->wr_tx_dma_addr) { 481 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, 482 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 483 DMA_TO_DEVICE); 484 lnk->wr_tx_dma_addr = 0; 485 } 486 } 487 488 void smc_wr_free_link_mem(struct smc_link *lnk) 489 { 490 kfree(lnk->wr_tx_pends); 491 lnk->wr_tx_pends = NULL; 492 kfree(lnk->wr_tx_mask); 493 lnk->wr_tx_mask = NULL; 494 kfree(lnk->wr_tx_sges); 495 lnk->wr_tx_sges = NULL; 496 kfree(lnk->wr_rx_sges); 497 lnk->wr_rx_sges = NULL; 498 kfree(lnk->wr_rx_ibs); 499 lnk->wr_rx_ibs = NULL; 500 kfree(lnk->wr_tx_ibs); 501 lnk->wr_tx_ibs = NULL; 502 kfree(lnk->wr_tx_bufs); 503 lnk->wr_tx_bufs = NULL; 504 kfree(lnk->wr_rx_bufs); 505 lnk->wr_rx_bufs = NULL; 506 } 507 508 int smc_wr_alloc_link_mem(struct smc_link *link) 509 { 510 /* allocate link related memory */ 511 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); 512 if (!link->wr_tx_bufs) 513 goto no_mem; 514 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, 515 GFP_KERNEL); 516 if (!link->wr_rx_bufs) 517 goto no_mem_wr_tx_bufs; 518 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), 519 GFP_KERNEL); 520 if (!link->wr_tx_ibs) 521 goto no_mem_wr_rx_bufs; 522 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, 523 sizeof(link->wr_rx_ibs[0]), 524 GFP_KERNEL); 525 if (!link->wr_rx_ibs) 526 goto no_mem_wr_tx_ibs; 527 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), 528 GFP_KERNEL); 529 if (!link->wr_tx_sges) 530 goto no_mem_wr_rx_ibs; 531 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, 532 sizeof(link->wr_rx_sges[0]), 533 GFP_KERNEL); 534 if (!link->wr_rx_sges) 535 goto no_mem_wr_tx_sges; 536 link->wr_tx_mask = kzalloc( 537 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*link->wr_tx_mask), 538 GFP_KERNEL); 539 if (!link->wr_tx_mask) 540 goto no_mem_wr_rx_sges; 541 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, 542 sizeof(link->wr_tx_pends[0]), 543 GFP_KERNEL); 544 if (!link->wr_tx_pends) 545 goto no_mem_wr_tx_mask; 546 return 0; 547 548 no_mem_wr_tx_mask: 549 kfree(link->wr_tx_mask); 550 no_mem_wr_rx_sges: 551 kfree(link->wr_rx_sges); 552 no_mem_wr_tx_sges: 553 kfree(link->wr_tx_sges); 554 no_mem_wr_rx_ibs: 555 kfree(link->wr_rx_ibs); 556 no_mem_wr_tx_ibs: 557 kfree(link->wr_tx_ibs); 558 no_mem_wr_rx_bufs: 559 kfree(link->wr_rx_bufs); 560 no_mem_wr_tx_bufs: 561 kfree(link->wr_tx_bufs); 562 no_mem: 563 return -ENOMEM; 564 } 565 566 void smc_wr_remove_dev(struct smc_ib_device *smcibdev) 567 { 568 tasklet_kill(&smcibdev->recv_tasklet); 569 tasklet_kill(&smcibdev->send_tasklet); 570 } 571 572 void smc_wr_add_dev(struct smc_ib_device *smcibdev) 573 { 574 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn, 575 (unsigned long)smcibdev); 576 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn, 577 (unsigned long)smcibdev); 578 } 579 580 int smc_wr_create_link(struct smc_link *lnk) 581 { 582 struct ib_device *ibdev = lnk->smcibdev->ibdev; 583 int rc = 0; 584 585 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); 586 lnk->wr_rx_id = 0; 587 lnk->wr_rx_dma_addr = ib_dma_map_single( 588 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 589 DMA_FROM_DEVICE); 590 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { 591 lnk->wr_rx_dma_addr = 0; 592 rc = -EIO; 593 goto out; 594 } 595 lnk->wr_tx_dma_addr = ib_dma_map_single( 596 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, 597 DMA_TO_DEVICE); 598 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { 599 rc = -EIO; 600 goto dma_unmap; 601 } 602 smc_wr_init_sge(lnk); 603 memset(lnk->wr_tx_mask, 0, 604 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask)); 605 return rc; 606 607 dma_unmap: 608 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, 609 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, 610 DMA_FROM_DEVICE); 611 lnk->wr_rx_dma_addr = 0; 612 out: 613 return rc; 614 } 615