1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2 3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 4 /* Copyright (c) 2008-2019, IBM Corporation */ 5 6 #include <linux/errno.h> 7 #include <linux/types.h> 8 #include <linux/net.h> 9 #include <linux/scatterlist.h> 10 #include <linux/llist.h> 11 #include <asm/barrier.h> 12 #include <net/tcp.h> 13 #include <trace/events/sock.h> 14 15 #include "siw.h" 16 #include "siw_verbs.h" 17 #include "siw_mem.h" 18 19 static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = { 20 [SIW_QP_STATE_IDLE] = "IDLE", 21 [SIW_QP_STATE_RTR] = "RTR", 22 [SIW_QP_STATE_RTS] = "RTS", 23 [SIW_QP_STATE_CLOSING] = "CLOSING", 24 [SIW_QP_STATE_TERMINATE] = "TERMINATE", 25 [SIW_QP_STATE_ERROR] = "ERROR" 26 }; 27 28 /* 29 * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a 30 * per-RDMAP message basis. Please keep order of initializer. All MPA len 31 * is initialized to minimum packet size. 32 */ 33 struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = { 34 { /* RDMAP_RDMA_WRITE */ 35 .hdr_len = sizeof(struct iwarp_rdma_write), 36 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2), 37 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST | 38 cpu_to_be16(DDP_VERSION << 8) | 39 cpu_to_be16(RDMAP_VERSION << 6) | 40 cpu_to_be16(RDMAP_RDMA_WRITE), 41 .rx_data = siw_proc_write }, 42 { /* RDMAP_RDMA_READ_REQ */ 43 .hdr_len = sizeof(struct iwarp_rdma_rreq), 44 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2), 45 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 46 cpu_to_be16(RDMAP_VERSION << 6) | 47 cpu_to_be16(RDMAP_RDMA_READ_REQ), 48 .rx_data = siw_proc_rreq }, 49 { /* RDMAP_RDMA_READ_RESP */ 50 .hdr_len = sizeof(struct iwarp_rdma_rresp), 51 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2), 52 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST | 53 cpu_to_be16(DDP_VERSION << 8) | 54 cpu_to_be16(RDMAP_VERSION << 6) | 55 cpu_to_be16(RDMAP_RDMA_READ_RESP), 56 .rx_data = siw_proc_rresp }, 57 { /* RDMAP_SEND */ 58 .hdr_len = sizeof(struct iwarp_send), 59 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2), 60 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 61 cpu_to_be16(RDMAP_VERSION << 6) | 62 cpu_to_be16(RDMAP_SEND), 63 .rx_data = siw_proc_send }, 64 { /* RDMAP_SEND_INVAL */ 65 .hdr_len = sizeof(struct iwarp_send_inv), 66 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2), 67 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 68 cpu_to_be16(RDMAP_VERSION << 6) | 69 cpu_to_be16(RDMAP_SEND_INVAL), 70 .rx_data = siw_proc_send }, 71 { /* RDMAP_SEND_SE */ 72 .hdr_len = sizeof(struct iwarp_send), 73 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2), 74 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 75 cpu_to_be16(RDMAP_VERSION << 6) | 76 cpu_to_be16(RDMAP_SEND_SE), 77 .rx_data = siw_proc_send }, 78 { /* RDMAP_SEND_SE_INVAL */ 79 .hdr_len = sizeof(struct iwarp_send_inv), 80 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2), 81 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 82 cpu_to_be16(RDMAP_VERSION << 6) | 83 cpu_to_be16(RDMAP_SEND_SE_INVAL), 84 .rx_data = siw_proc_send }, 85 { /* RDMAP_TERMINATE */ 86 .hdr_len = sizeof(struct iwarp_terminate), 87 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2), 88 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 89 cpu_to_be16(RDMAP_VERSION << 6) | 90 cpu_to_be16(RDMAP_TERMINATE), 91 .rx_data = siw_proc_terminate } 92 }; 93 94 void siw_qp_llp_data_ready(struct sock *sk) 95 { 96 struct siw_qp *qp; 97 98 trace_sk_data_ready(sk); 99 100 read_lock(&sk->sk_callback_lock); 101 102 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk))) 103 goto done; 104 105 qp = sk_to_qp(sk); 106 107 if (likely(!qp->rx_stream.rx_suspend && 108 down_read_trylock(&qp->state_lock))) { 109 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 }; 110 111 if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) 112 /* 113 * Implements data receive operation during 114 * socket callback. TCP gracefully catches 115 * the case where there is nothing to receive 116 * (not calling siw_tcp_rx_data() then). 117 */ 118 tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data); 119 120 up_read(&qp->state_lock); 121 } else { 122 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n", 123 qp->rx_stream.rx_suspend); 124 } 125 done: 126 read_unlock(&sk->sk_callback_lock); 127 } 128 129 void siw_qp_llp_close(struct siw_qp *qp) 130 { 131 siw_dbg_qp(qp, "enter llp close, state = %s\n", 132 siw_qp_state_to_string[qp->attrs.state]); 133 134 down_write(&qp->state_lock); 135 136 qp->rx_stream.rx_suspend = 1; 137 qp->tx_ctx.tx_suspend = 1; 138 qp->attrs.sk = NULL; 139 140 switch (qp->attrs.state) { 141 case SIW_QP_STATE_RTS: 142 case SIW_QP_STATE_RTR: 143 case SIW_QP_STATE_IDLE: 144 case SIW_QP_STATE_TERMINATE: 145 qp->attrs.state = SIW_QP_STATE_ERROR; 146 break; 147 /* 148 * SIW_QP_STATE_CLOSING: 149 * 150 * This is a forced close. shall the QP be moved to 151 * ERROR or IDLE ? 152 */ 153 case SIW_QP_STATE_CLOSING: 154 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) 155 qp->attrs.state = SIW_QP_STATE_ERROR; 156 else 157 qp->attrs.state = SIW_QP_STATE_IDLE; 158 break; 159 160 default: 161 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n", 162 siw_qp_state_to_string[qp->attrs.state]); 163 break; 164 } 165 siw_sq_flush(qp); 166 siw_rq_flush(qp); 167 168 /* 169 * Dereference closing CEP 170 */ 171 if (qp->cep) { 172 siw_cep_put(qp->cep); 173 qp->cep = NULL; 174 } 175 176 up_write(&qp->state_lock); 177 178 siw_dbg_qp(qp, "llp close exit: state %s\n", 179 siw_qp_state_to_string[qp->attrs.state]); 180 } 181 182 /* 183 * socket callback routine informing about newly available send space. 184 * Function schedules SQ work for processing SQ items. 185 */ 186 void siw_qp_llp_write_space(struct sock *sk) 187 { 188 struct siw_cep *cep; 189 190 read_lock(&sk->sk_callback_lock); 191 192 cep = sk_to_cep(sk); 193 if (cep) { 194 cep->sk_write_space(sk); 195 196 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) 197 (void)siw_sq_start(cep->qp); 198 } 199 200 read_unlock(&sk->sk_callback_lock); 201 } 202 203 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size) 204 { 205 if (irq_size) { 206 irq_size = roundup_pow_of_two(irq_size); 207 qp->irq = vcalloc(irq_size, sizeof(struct siw_sqe)); 208 if (!qp->irq) { 209 qp->attrs.irq_size = 0; 210 return -ENOMEM; 211 } 212 } 213 if (orq_size) { 214 orq_size = roundup_pow_of_two(orq_size); 215 qp->orq = vcalloc(orq_size, sizeof(struct siw_sqe)); 216 if (!qp->orq) { 217 qp->attrs.orq_size = 0; 218 qp->attrs.irq_size = 0; 219 vfree(qp->irq); 220 return -ENOMEM; 221 } 222 } 223 qp->attrs.irq_size = irq_size; 224 qp->attrs.orq_size = orq_size; 225 siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size); 226 return 0; 227 } 228 229 static int siw_qp_enable_crc(struct siw_qp *qp) 230 { 231 struct siw_rx_stream *c_rx = &qp->rx_stream; 232 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; 233 int size; 234 235 if (siw_crypto_shash == NULL) 236 return -ENOENT; 237 238 size = crypto_shash_descsize(siw_crypto_shash) + 239 sizeof(struct shash_desc); 240 241 c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); 242 c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); 243 if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) { 244 kfree(c_tx->mpa_crc_hd); 245 kfree(c_rx->mpa_crc_hd); 246 c_tx->mpa_crc_hd = NULL; 247 c_rx->mpa_crc_hd = NULL; 248 return -ENOMEM; 249 } 250 c_tx->mpa_crc_hd->tfm = siw_crypto_shash; 251 c_rx->mpa_crc_hd->tfm = siw_crypto_shash; 252 253 return 0; 254 } 255 256 /* 257 * Send a non signalled READ or WRITE to peer side as negotiated 258 * with MPAv2 P2P setup protocol. The work request is only created 259 * as a current active WR and does not consume Send Queue space. 260 * 261 * Caller must hold QP state lock. 262 */ 263 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl) 264 { 265 struct siw_wqe *wqe = tx_wqe(qp); 266 unsigned long flags; 267 int rv = 0; 268 269 spin_lock_irqsave(&qp->sq_lock, flags); 270 271 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { 272 spin_unlock_irqrestore(&qp->sq_lock, flags); 273 return -EIO; 274 } 275 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 276 277 wqe->wr_status = SIW_WR_QUEUED; 278 wqe->sqe.flags = 0; 279 wqe->sqe.num_sge = 1; 280 wqe->sqe.sge[0].length = 0; 281 wqe->sqe.sge[0].laddr = 0; 282 wqe->sqe.sge[0].lkey = 0; 283 /* 284 * While it must not be checked for inbound zero length 285 * READ/WRITE, some HW may treat STag 0 special. 286 */ 287 wqe->sqe.rkey = 1; 288 wqe->sqe.raddr = 0; 289 wqe->processed = 0; 290 291 if (ctrl & MPA_V2_RDMA_WRITE_RTR) 292 wqe->sqe.opcode = SIW_OP_WRITE; 293 else if (ctrl & MPA_V2_RDMA_READ_RTR) { 294 struct siw_sqe *rreq = NULL; 295 296 wqe->sqe.opcode = SIW_OP_READ; 297 298 spin_lock(&qp->orq_lock); 299 300 if (qp->attrs.orq_size) 301 rreq = orq_get_free(qp); 302 if (rreq) { 303 siw_read_to_orq(rreq, &wqe->sqe); 304 qp->orq_put++; 305 } else 306 rv = -EIO; 307 308 spin_unlock(&qp->orq_lock); 309 } else 310 rv = -EINVAL; 311 312 if (rv) 313 wqe->wr_status = SIW_WR_IDLE; 314 315 spin_unlock_irqrestore(&qp->sq_lock, flags); 316 317 if (!rv) 318 rv = siw_sq_start(qp); 319 320 return rv; 321 } 322 323 /* 324 * Map memory access error to DDP tagged error 325 */ 326 enum ddp_ecode siw_tagged_error(enum siw_access_state state) 327 { 328 switch (state) { 329 case E_STAG_INVALID: 330 return DDP_ECODE_T_INVALID_STAG; 331 case E_BASE_BOUNDS: 332 return DDP_ECODE_T_BASE_BOUNDS; 333 case E_PD_MISMATCH: 334 return DDP_ECODE_T_STAG_NOT_ASSOC; 335 case E_ACCESS_PERM: 336 /* 337 * RFC 5041 (DDP) lacks an ecode for insufficient access 338 * permissions. 'Invalid STag' seem to be the closest 339 * match though. 340 */ 341 return DDP_ECODE_T_INVALID_STAG; 342 default: 343 WARN_ON(1); 344 return DDP_ECODE_T_INVALID_STAG; 345 } 346 } 347 348 /* 349 * Map memory access error to RDMAP protection error 350 */ 351 enum rdmap_ecode siw_rdmap_error(enum siw_access_state state) 352 { 353 switch (state) { 354 case E_STAG_INVALID: 355 return RDMAP_ECODE_INVALID_STAG; 356 case E_BASE_BOUNDS: 357 return RDMAP_ECODE_BASE_BOUNDS; 358 case E_PD_MISMATCH: 359 return RDMAP_ECODE_STAG_NOT_ASSOC; 360 case E_ACCESS_PERM: 361 return RDMAP_ECODE_ACCESS_RIGHTS; 362 default: 363 return RDMAP_ECODE_UNSPECIFIED; 364 } 365 } 366 367 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype, 368 u8 ecode, int in_tx) 369 { 370 if (!qp->term_info.valid) { 371 memset(&qp->term_info, 0, sizeof(qp->term_info)); 372 qp->term_info.layer = layer; 373 qp->term_info.etype = etype; 374 qp->term_info.ecode = ecode; 375 qp->term_info.in_tx = in_tx; 376 qp->term_info.valid = 1; 377 } 378 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n", 379 layer, etype, ecode, in_tx ? "yes" : "no"); 380 } 381 382 /* 383 * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581. 384 * Sending TERMINATE messages is best effort - such messages 385 * can only be send if the QP is still connected and it does 386 * not have another outbound message in-progress, i.e. the 387 * TERMINATE message must not interfer with an incomplete current 388 * transmit operation. 389 */ 390 void siw_send_terminate(struct siw_qp *qp) 391 { 392 struct kvec iov[3]; 393 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR }; 394 struct iwarp_terminate *term = NULL; 395 union iwarp_hdr *err_hdr = NULL; 396 struct socket *s = qp->attrs.sk; 397 struct siw_rx_stream *srx = &qp->rx_stream; 398 union iwarp_hdr *rx_hdr = &srx->hdr; 399 u32 crc = 0; 400 int num_frags, len_terminate, rv; 401 402 if (!qp->term_info.valid) 403 return; 404 405 qp->term_info.valid = 0; 406 407 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { 408 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n", 409 tx_type(tx_wqe(qp))); 410 return; 411 } 412 if (!s && qp->cep) 413 /* QP not yet in RTS. Take socket from connection end point */ 414 s = qp->cep->sock; 415 416 if (!s) { 417 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n"); 418 return; 419 } 420 421 term = kzalloc(sizeof(*term), GFP_KERNEL); 422 if (!term) 423 return; 424 425 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE); 426 term->ddp_mo = 0; 427 term->ddp_msn = cpu_to_be32(1); 428 429 iov[0].iov_base = term; 430 iov[0].iov_len = sizeof(*term); 431 432 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || 433 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && 434 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { 435 err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL); 436 if (!err_hdr) { 437 kfree(term); 438 return; 439 } 440 } 441 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl, 442 sizeof(struct iwarp_ctrl)); 443 444 __rdmap_term_set_layer(term, qp->term_info.layer); 445 __rdmap_term_set_etype(term, qp->term_info.etype); 446 __rdmap_term_set_ecode(term, qp->term_info.ecode); 447 448 switch (qp->term_info.layer) { 449 case TERM_ERROR_LAYER_RDMAP: 450 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) 451 /* No additional DDP/RDMAP header to be included */ 452 break; 453 454 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { 455 /* 456 * Complete RDMAP frame will get attached, and 457 * DDP segment length is valid 458 */ 459 term->flag_m = 1; 460 term->flag_d = 1; 461 term->flag_r = 1; 462 463 if (qp->term_info.in_tx) { 464 struct iwarp_rdma_rreq *rreq; 465 struct siw_wqe *wqe = tx_wqe(qp); 466 467 /* Inbound RREQ error, detected during 468 * RRESP creation. Take state from 469 * current TX work queue element to 470 * reconstruct peers RREQ. 471 */ 472 rreq = (struct iwarp_rdma_rreq *)err_hdr; 473 474 memcpy(&rreq->ctrl, 475 &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl, 476 sizeof(struct iwarp_ctrl)); 477 478 rreq->rsvd = 0; 479 rreq->ddp_qn = 480 htonl(RDMAP_UNTAGGED_QN_RDMA_READ); 481 482 /* Provide RREQ's MSN as kept aside */ 483 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length); 484 485 rreq->ddp_mo = htonl(wqe->processed); 486 rreq->sink_stag = htonl(wqe->sqe.rkey); 487 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr); 488 rreq->read_size = htonl(wqe->sqe.sge[0].length); 489 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey); 490 rreq->source_to = 491 cpu_to_be64(wqe->sqe.sge[0].laddr); 492 493 iov[1].iov_base = rreq; 494 iov[1].iov_len = sizeof(*rreq); 495 496 rx_hdr = (union iwarp_hdr *)rreq; 497 } else { 498 /* Take RDMAP/DDP information from 499 * current (failed) inbound frame. 500 */ 501 iov[1].iov_base = rx_hdr; 502 503 if (__rdmap_get_opcode(&rx_hdr->ctrl) == 504 RDMAP_RDMA_READ_REQ) 505 iov[1].iov_len = 506 sizeof(struct iwarp_rdma_rreq); 507 else /* SEND type */ 508 iov[1].iov_len = 509 sizeof(struct iwarp_send); 510 } 511 } else { 512 /* Do not report DDP hdr information if packet 513 * layout is unknown 514 */ 515 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || 516 (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) 517 break; 518 519 iov[1].iov_base = rx_hdr; 520 521 /* Only DDP frame will get attached */ 522 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) 523 iov[1].iov_len = 524 sizeof(struct iwarp_rdma_write); 525 else 526 iov[1].iov_len = sizeof(struct iwarp_send); 527 528 term->flag_m = 1; 529 term->flag_d = 1; 530 } 531 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len); 532 break; 533 534 case TERM_ERROR_LAYER_DDP: 535 /* Report error encountered while DDP processing. 536 * This can only happen as a result of inbound 537 * DDP processing 538 */ 539 540 /* Do not report DDP hdr information if packet 541 * layout is unknown 542 */ 543 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && 544 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || 545 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && 546 (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) 547 break; 548 549 iov[1].iov_base = rx_hdr; 550 551 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) 552 iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged); 553 else 554 iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged); 555 556 term->flag_m = 1; 557 term->flag_d = 1; 558 break; 559 560 default: 561 break; 562 } 563 if (term->flag_m || term->flag_d || term->flag_r) { 564 iov[2].iov_base = &crc; 565 iov[2].iov_len = sizeof(crc); 566 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE; 567 num_frags = 3; 568 } else { 569 iov[1].iov_base = &crc; 570 iov[1].iov_len = sizeof(crc); 571 len_terminate = sizeof(*term) + MPA_CRC_SIZE; 572 num_frags = 2; 573 } 574 575 /* Adjust DDP Segment Length parameter, if valid */ 576 if (term->flag_m) { 577 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len); 578 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl); 579 580 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE; 581 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len); 582 } 583 584 term->ctrl.mpa_len = 585 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE)); 586 if (qp->tx_ctx.mpa_crc_hd) { 587 crypto_shash_init(qp->tx_ctx.mpa_crc_hd); 588 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, 589 (u8 *)iov[0].iov_base, 590 iov[0].iov_len)) 591 goto out; 592 593 if (num_frags == 3) { 594 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, 595 (u8 *)iov[1].iov_base, 596 iov[1].iov_len)) 597 goto out; 598 } 599 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc); 600 } 601 602 rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate); 603 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n", 604 rv == len_terminate ? "success" : "failure", 605 __rdmap_term_layer(term), __rdmap_term_etype(term), 606 __rdmap_term_ecode(term), rv); 607 out: 608 kfree(term); 609 kfree(err_hdr); 610 } 611 612 /* 613 * Handle all attrs other than state 614 */ 615 static void siw_qp_modify_nonstate(struct siw_qp *qp, 616 struct siw_qp_attrs *attrs, 617 enum siw_qp_attr_mask mask) 618 { 619 if (mask & SIW_QP_ATTR_ACCESS_FLAGS) { 620 if (attrs->flags & SIW_RDMA_BIND_ENABLED) 621 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; 622 else 623 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; 624 625 if (attrs->flags & SIW_RDMA_WRITE_ENABLED) 626 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; 627 else 628 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; 629 630 if (attrs->flags & SIW_RDMA_READ_ENABLED) 631 qp->attrs.flags |= SIW_RDMA_READ_ENABLED; 632 else 633 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; 634 } 635 } 636 637 static int siw_qp_nextstate_from_idle(struct siw_qp *qp, 638 struct siw_qp_attrs *attrs, 639 enum siw_qp_attr_mask mask) 640 { 641 int rv = 0; 642 643 switch (attrs->state) { 644 case SIW_QP_STATE_RTS: 645 if (attrs->flags & SIW_MPA_CRC) { 646 rv = siw_qp_enable_crc(qp); 647 if (rv) 648 break; 649 } 650 if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) { 651 siw_dbg_qp(qp, "no socket\n"); 652 rv = -EINVAL; 653 break; 654 } 655 if (!(mask & SIW_QP_ATTR_MPA)) { 656 siw_dbg_qp(qp, "no MPA\n"); 657 rv = -EINVAL; 658 break; 659 } 660 /* 661 * Initialize iWARP TX state 662 */ 663 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; 664 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; 665 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; 666 667 /* 668 * Initialize iWARP RX state 669 */ 670 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; 671 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; 672 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; 673 674 /* 675 * init IRD free queue, caller has already checked 676 * limits. 677 */ 678 rv = siw_qp_readq_init(qp, attrs->irq_size, 679 attrs->orq_size); 680 if (rv) 681 break; 682 683 qp->attrs.sk = attrs->sk; 684 qp->attrs.state = SIW_QP_STATE_RTS; 685 686 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n", 687 attrs->flags & SIW_MPA_CRC ? "y" : "n", 688 qp->attrs.orq_size, qp->attrs.irq_size); 689 break; 690 691 case SIW_QP_STATE_ERROR: 692 siw_rq_flush(qp); 693 qp->attrs.state = SIW_QP_STATE_ERROR; 694 if (qp->cep) { 695 siw_cep_put(qp->cep); 696 qp->cep = NULL; 697 } 698 break; 699 700 default: 701 break; 702 } 703 return rv; 704 } 705 706 static int siw_qp_nextstate_from_rts(struct siw_qp *qp, 707 struct siw_qp_attrs *attrs) 708 { 709 int drop_conn = 0; 710 711 switch (attrs->state) { 712 case SIW_QP_STATE_CLOSING: 713 /* 714 * Verbs: move to IDLE if SQ and ORQ are empty. 715 * Move to ERROR otherwise. But first of all we must 716 * close the connection. So we keep CLOSING or ERROR 717 * as a transient state, schedule connection drop work 718 * and wait for the socket state change upcall to 719 * come back closed. 720 */ 721 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { 722 qp->attrs.state = SIW_QP_STATE_CLOSING; 723 } else { 724 qp->attrs.state = SIW_QP_STATE_ERROR; 725 siw_sq_flush(qp); 726 } 727 siw_rq_flush(qp); 728 729 drop_conn = 1; 730 break; 731 732 case SIW_QP_STATE_TERMINATE: 733 qp->attrs.state = SIW_QP_STATE_TERMINATE; 734 735 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, 736 RDMAP_ETYPE_CATASTROPHIC, 737 RDMAP_ECODE_UNSPECIFIED, 1); 738 drop_conn = 1; 739 break; 740 741 case SIW_QP_STATE_ERROR: 742 /* 743 * This is an emergency close. 744 * 745 * Any in progress transmit operation will get 746 * cancelled. 747 * This will likely result in a protocol failure, 748 * if a TX operation is in transit. The caller 749 * could unconditional wait to give the current 750 * operation a chance to complete. 751 * Esp., how to handle the non-empty IRQ case? 752 * The peer was asking for data transfer at a valid 753 * point in time. 754 */ 755 siw_sq_flush(qp); 756 siw_rq_flush(qp); 757 qp->attrs.state = SIW_QP_STATE_ERROR; 758 drop_conn = 1; 759 break; 760 761 default: 762 break; 763 } 764 return drop_conn; 765 } 766 767 static void siw_qp_nextstate_from_term(struct siw_qp *qp, 768 struct siw_qp_attrs *attrs) 769 { 770 switch (attrs->state) { 771 case SIW_QP_STATE_ERROR: 772 siw_rq_flush(qp); 773 qp->attrs.state = SIW_QP_STATE_ERROR; 774 775 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) 776 siw_sq_flush(qp); 777 break; 778 779 default: 780 break; 781 } 782 } 783 784 static int siw_qp_nextstate_from_close(struct siw_qp *qp, 785 struct siw_qp_attrs *attrs) 786 { 787 int rv = 0; 788 789 switch (attrs->state) { 790 case SIW_QP_STATE_IDLE: 791 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); 792 qp->attrs.state = SIW_QP_STATE_IDLE; 793 break; 794 795 case SIW_QP_STATE_CLOSING: 796 /* 797 * The LLP may already moved the QP to closing 798 * due to graceful peer close init 799 */ 800 break; 801 802 case SIW_QP_STATE_ERROR: 803 /* 804 * QP was moved to CLOSING by LLP event 805 * not yet seen by user. 806 */ 807 qp->attrs.state = SIW_QP_STATE_ERROR; 808 809 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) 810 siw_sq_flush(qp); 811 812 siw_rq_flush(qp); 813 break; 814 815 default: 816 siw_dbg_qp(qp, "state transition undefined: %s => %s\n", 817 siw_qp_state_to_string[qp->attrs.state], 818 siw_qp_state_to_string[attrs->state]); 819 820 rv = -ECONNABORTED; 821 } 822 return rv; 823 } 824 825 /* 826 * Caller must hold qp->state_lock 827 */ 828 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs, 829 enum siw_qp_attr_mask mask) 830 { 831 int drop_conn = 0, rv = 0; 832 833 if (!mask) 834 return 0; 835 836 siw_dbg_qp(qp, "state: %s => %s\n", 837 siw_qp_state_to_string[qp->attrs.state], 838 siw_qp_state_to_string[attrs->state]); 839 840 if (mask != SIW_QP_ATTR_STATE) 841 siw_qp_modify_nonstate(qp, attrs, mask); 842 843 if (!(mask & SIW_QP_ATTR_STATE)) 844 return 0; 845 846 switch (qp->attrs.state) { 847 case SIW_QP_STATE_IDLE: 848 case SIW_QP_STATE_RTR: 849 rv = siw_qp_nextstate_from_idle(qp, attrs, mask); 850 break; 851 852 case SIW_QP_STATE_RTS: 853 drop_conn = siw_qp_nextstate_from_rts(qp, attrs); 854 break; 855 856 case SIW_QP_STATE_TERMINATE: 857 siw_qp_nextstate_from_term(qp, attrs); 858 break; 859 860 case SIW_QP_STATE_CLOSING: 861 siw_qp_nextstate_from_close(qp, attrs); 862 break; 863 default: 864 break; 865 } 866 if (drop_conn) 867 siw_qp_cm_drop(qp, 0); 868 869 return rv; 870 } 871 872 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe) 873 { 874 rreq->id = sqe->id; 875 rreq->opcode = sqe->opcode; 876 rreq->sge[0].laddr = sqe->sge[0].laddr; 877 rreq->sge[0].length = sqe->sge[0].length; 878 rreq->sge[0].lkey = sqe->sge[0].lkey; 879 rreq->sge[1].lkey = sqe->sge[1].lkey; 880 rreq->flags = sqe->flags | SIW_WQE_VALID; 881 rreq->num_sge = 1; 882 } 883 884 static int siw_activate_tx_from_sq(struct siw_qp *qp) 885 { 886 struct siw_sqe *sqe; 887 struct siw_wqe *wqe = tx_wqe(qp); 888 int rv = 1; 889 890 sqe = sq_get_next(qp); 891 if (!sqe) 892 return 0; 893 894 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 895 wqe->wr_status = SIW_WR_QUEUED; 896 897 /* First copy SQE to kernel private memory */ 898 memcpy(&wqe->sqe, sqe, sizeof(*sqe)); 899 900 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) { 901 rv = -EINVAL; 902 goto out; 903 } 904 if (wqe->sqe.flags & SIW_WQE_INLINE) { 905 if (wqe->sqe.opcode != SIW_OP_SEND && 906 wqe->sqe.opcode != SIW_OP_WRITE) { 907 rv = -EINVAL; 908 goto out; 909 } 910 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) { 911 rv = -EINVAL; 912 goto out; 913 } 914 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1]; 915 wqe->sqe.sge[0].lkey = 0; 916 wqe->sqe.num_sge = 1; 917 } 918 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) { 919 /* A READ cannot be fenced */ 920 if (unlikely(wqe->sqe.opcode == SIW_OP_READ || 921 wqe->sqe.opcode == 922 SIW_OP_READ_LOCAL_INV)) { 923 siw_dbg_qp(qp, "cannot fence read\n"); 924 rv = -EINVAL; 925 goto out; 926 } 927 spin_lock(&qp->orq_lock); 928 929 if (qp->attrs.orq_size && !siw_orq_empty(qp)) { 930 qp->tx_ctx.orq_fence = 1; 931 rv = 0; 932 } 933 spin_unlock(&qp->orq_lock); 934 935 } else if (wqe->sqe.opcode == SIW_OP_READ || 936 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) { 937 struct siw_sqe *rreq; 938 939 if (unlikely(!qp->attrs.orq_size)) { 940 /* We negotiated not to send READ req's */ 941 rv = -EINVAL; 942 goto out; 943 } 944 wqe->sqe.num_sge = 1; 945 946 spin_lock(&qp->orq_lock); 947 948 rreq = orq_get_free(qp); 949 if (rreq) { 950 /* 951 * Make an immediate copy in ORQ to be ready 952 * to process loopback READ reply 953 */ 954 siw_read_to_orq(rreq, &wqe->sqe); 955 qp->orq_put++; 956 } else { 957 qp->tx_ctx.orq_fence = 1; 958 rv = 0; 959 } 960 spin_unlock(&qp->orq_lock); 961 } 962 963 /* Clear SQE, can be re-used by application */ 964 smp_store_mb(sqe->flags, 0); 965 qp->sq_get++; 966 out: 967 if (unlikely(rv < 0)) { 968 siw_dbg_qp(qp, "error %d\n", rv); 969 wqe->wr_status = SIW_WR_IDLE; 970 } 971 return rv; 972 } 973 974 /* 975 * Must be called with SQ locked. 976 * To avoid complete SQ starvation by constant inbound READ requests, 977 * the active IRQ will not be served after qp->irq_burst, if the 978 * SQ has pending work. 979 */ 980 int siw_activate_tx(struct siw_qp *qp) 981 { 982 struct siw_sqe *irqe; 983 struct siw_wqe *wqe = tx_wqe(qp); 984 985 if (!qp->attrs.irq_size) 986 return siw_activate_tx_from_sq(qp); 987 988 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; 989 990 if (!(irqe->flags & SIW_WQE_VALID)) 991 return siw_activate_tx_from_sq(qp); 992 993 /* 994 * Avoid local WQE processing starvation in case 995 * of constant inbound READ request stream 996 */ 997 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { 998 qp->irq_burst = 0; 999 return siw_activate_tx_from_sq(qp); 1000 } 1001 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 1002 wqe->wr_status = SIW_WR_QUEUED; 1003 1004 /* start READ RESPONSE */ 1005 wqe->sqe.opcode = SIW_OP_READ_RESPONSE; 1006 wqe->sqe.flags = 0; 1007 if (irqe->num_sge) { 1008 wqe->sqe.num_sge = 1; 1009 wqe->sqe.sge[0].length = irqe->sge[0].length; 1010 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr; 1011 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey; 1012 } else { 1013 wqe->sqe.num_sge = 0; 1014 } 1015 1016 /* Retain original RREQ's message sequence number for 1017 * potential error reporting cases. 1018 */ 1019 wqe->sqe.sge[1].length = irqe->sge[1].length; 1020 1021 wqe->sqe.rkey = irqe->rkey; 1022 wqe->sqe.raddr = irqe->raddr; 1023 1024 wqe->processed = 0; 1025 qp->irq_get++; 1026 1027 /* mark current IRQ entry free */ 1028 smp_store_mb(irqe->flags, 0); 1029 1030 return 1; 1031 } 1032 1033 /* 1034 * Check if current CQ state qualifies for calling CQ completion 1035 * handler. Must be called with CQ lock held. 1036 */ 1037 static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags) 1038 { 1039 u32 cq_notify; 1040 1041 if (!cq->base_cq.comp_handler) 1042 return false; 1043 1044 /* Read application shared notification state */ 1045 cq_notify = READ_ONCE(cq->notify->flags); 1046 1047 if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) || 1048 ((cq_notify & SIW_NOTIFY_SOLICITED) && 1049 (flags & SIW_WQE_SOLICITED))) { 1050 /* 1051 * CQ notification is one-shot: Since the 1052 * current CQE causes user notification, 1053 * the CQ gets dis-aremd and must be re-aremd 1054 * by the user for a new notification. 1055 */ 1056 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT); 1057 1058 return true; 1059 } 1060 return false; 1061 } 1062 1063 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, 1064 enum siw_wc_status status) 1065 { 1066 struct siw_cq *cq = qp->scq; 1067 int rv = 0; 1068 1069 if (cq) { 1070 u32 sqe_flags = sqe->flags; 1071 struct siw_cqe *cqe; 1072 u32 idx; 1073 unsigned long flags; 1074 1075 spin_lock_irqsave(&cq->lock, flags); 1076 1077 idx = cq->cq_put % cq->num_cqe; 1078 cqe = &cq->queue[idx]; 1079 1080 if (!READ_ONCE(cqe->flags)) { 1081 bool notify; 1082 1083 cqe->id = sqe->id; 1084 cqe->opcode = sqe->opcode; 1085 cqe->status = status; 1086 cqe->imm_data = 0; 1087 cqe->bytes = bytes; 1088 1089 if (rdma_is_kernel_res(&cq->base_cq.res)) 1090 cqe->base_qp = &qp->base_qp; 1091 else 1092 cqe->qp_id = qp_id(qp); 1093 1094 /* mark CQE valid for application */ 1095 WRITE_ONCE(cqe->flags, SIW_WQE_VALID); 1096 /* recycle SQE */ 1097 smp_store_mb(sqe->flags, 0); 1098 1099 cq->cq_put++; 1100 notify = siw_cq_notify_now(cq, sqe_flags); 1101 1102 spin_unlock_irqrestore(&cq->lock, flags); 1103 1104 if (notify) { 1105 siw_dbg_cq(cq, "Call completion handler\n"); 1106 cq->base_cq.comp_handler(&cq->base_cq, 1107 cq->base_cq.cq_context); 1108 } 1109 } else { 1110 spin_unlock_irqrestore(&cq->lock, flags); 1111 rv = -ENOMEM; 1112 siw_cq_event(cq, IB_EVENT_CQ_ERR); 1113 } 1114 } else { 1115 /* recycle SQE */ 1116 smp_store_mb(sqe->flags, 0); 1117 } 1118 return rv; 1119 } 1120 1121 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, 1122 u32 inval_stag, enum siw_wc_status status) 1123 { 1124 struct siw_cq *cq = qp->rcq; 1125 int rv = 0; 1126 1127 if (cq) { 1128 struct siw_cqe *cqe; 1129 u32 idx; 1130 unsigned long flags; 1131 1132 spin_lock_irqsave(&cq->lock, flags); 1133 1134 idx = cq->cq_put % cq->num_cqe; 1135 cqe = &cq->queue[idx]; 1136 1137 if (!READ_ONCE(cqe->flags)) { 1138 bool notify; 1139 u8 cqe_flags = SIW_WQE_VALID; 1140 1141 cqe->id = rqe->id; 1142 cqe->opcode = SIW_OP_RECEIVE; 1143 cqe->status = status; 1144 cqe->imm_data = 0; 1145 cqe->bytes = bytes; 1146 1147 if (rdma_is_kernel_res(&cq->base_cq.res)) { 1148 cqe->base_qp = &qp->base_qp; 1149 if (inval_stag) { 1150 cqe_flags |= SIW_WQE_REM_INVAL; 1151 cqe->inval_stag = inval_stag; 1152 } 1153 } else { 1154 cqe->qp_id = qp_id(qp); 1155 } 1156 /* mark CQE valid for application */ 1157 WRITE_ONCE(cqe->flags, cqe_flags); 1158 /* recycle RQE */ 1159 smp_store_mb(rqe->flags, 0); 1160 1161 cq->cq_put++; 1162 notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED); 1163 1164 spin_unlock_irqrestore(&cq->lock, flags); 1165 1166 if (notify) { 1167 siw_dbg_cq(cq, "Call completion handler\n"); 1168 cq->base_cq.comp_handler(&cq->base_cq, 1169 cq->base_cq.cq_context); 1170 } 1171 } else { 1172 spin_unlock_irqrestore(&cq->lock, flags); 1173 rv = -ENOMEM; 1174 siw_cq_event(cq, IB_EVENT_CQ_ERR); 1175 } 1176 } else { 1177 /* recycle RQE */ 1178 smp_store_mb(rqe->flags, 0); 1179 } 1180 return rv; 1181 } 1182 1183 /* 1184 * siw_sq_flush() 1185 * 1186 * Flush SQ and ORRQ entries to CQ. 1187 * 1188 * Must be called with QP state write lock held. 1189 * Therefore, SQ and ORQ lock must not be taken. 1190 */ 1191 void siw_sq_flush(struct siw_qp *qp) 1192 { 1193 struct siw_sqe *sqe; 1194 struct siw_wqe *wqe = tx_wqe(qp); 1195 int async_event = 0; 1196 1197 /* 1198 * Start with completing any work currently on the ORQ 1199 */ 1200 while (qp->attrs.orq_size) { 1201 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; 1202 if (!READ_ONCE(sqe->flags)) 1203 break; 1204 1205 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1206 break; 1207 1208 WRITE_ONCE(sqe->flags, 0); 1209 qp->orq_get++; 1210 } 1211 /* 1212 * Flush an in-progress WQE if present 1213 */ 1214 if (wqe->wr_status != SIW_WR_IDLE) { 1215 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n", 1216 tx_type(wqe), wqe->wr_status); 1217 1218 siw_wqe_put_mem(wqe, tx_type(wqe)); 1219 1220 if (tx_type(wqe) != SIW_OP_READ_RESPONSE && 1221 ((tx_type(wqe) != SIW_OP_READ && 1222 tx_type(wqe) != SIW_OP_READ_LOCAL_INV) || 1223 wqe->wr_status == SIW_WR_QUEUED)) 1224 /* 1225 * An in-progress Read Request is already in 1226 * the ORQ 1227 */ 1228 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, 1229 SIW_WC_WR_FLUSH_ERR); 1230 1231 wqe->wr_status = SIW_WR_IDLE; 1232 } 1233 /* 1234 * Flush the Send Queue 1235 */ 1236 while (qp->attrs.sq_size) { 1237 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; 1238 if (!READ_ONCE(sqe->flags)) 1239 break; 1240 1241 async_event = 1; 1242 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1243 /* 1244 * Shall IB_EVENT_SQ_DRAINED be supressed if work 1245 * completion fails? 1246 */ 1247 break; 1248 1249 WRITE_ONCE(sqe->flags, 0); 1250 qp->sq_get++; 1251 } 1252 if (async_event) 1253 siw_qp_event(qp, IB_EVENT_SQ_DRAINED); 1254 } 1255 1256 /* 1257 * siw_rq_flush() 1258 * 1259 * Flush recv queue entries to CQ. Also 1260 * takes care of pending active tagged and untagged 1261 * inbound transfers, which have target memory 1262 * referenced. 1263 * 1264 * Must be called with QP state write lock held. 1265 * Therefore, RQ lock must not be taken. 1266 */ 1267 void siw_rq_flush(struct siw_qp *qp) 1268 { 1269 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; 1270 1271 /* 1272 * Flush an in-progress untagged operation if present 1273 */ 1274 if (wqe->wr_status != SIW_WR_IDLE) { 1275 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n", 1276 rx_type(wqe), wqe->wr_status); 1277 1278 siw_wqe_put_mem(wqe, rx_type(wqe)); 1279 1280 if (rx_type(wqe) == SIW_OP_RECEIVE) { 1281 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, 1282 0, SIW_WC_WR_FLUSH_ERR); 1283 } else if (rx_type(wqe) != SIW_OP_READ && 1284 rx_type(wqe) != SIW_OP_READ_RESPONSE && 1285 rx_type(wqe) != SIW_OP_WRITE) { 1286 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); 1287 } 1288 wqe->wr_status = SIW_WR_IDLE; 1289 } 1290 wqe = &qp->rx_tagged.wqe_active; 1291 1292 if (wqe->wr_status != SIW_WR_IDLE) { 1293 siw_wqe_put_mem(wqe, rx_type(wqe)); 1294 wqe->wr_status = SIW_WR_IDLE; 1295 } 1296 /* 1297 * Flush the Receive Queue 1298 */ 1299 while (qp->attrs.rq_size) { 1300 struct siw_rqe *rqe = 1301 &qp->recvq[qp->rq_get % qp->attrs.rq_size]; 1302 1303 if (!READ_ONCE(rqe->flags)) 1304 break; 1305 1306 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1307 break; 1308 1309 WRITE_ONCE(rqe->flags, 0); 1310 qp->rq_get++; 1311 } 1312 } 1313 1314 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) 1315 { 1316 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, 1317 GFP_KERNEL); 1318 1319 if (!rv) { 1320 kref_init(&qp->ref); 1321 qp->sdev = sdev; 1322 siw_dbg_qp(qp, "new QP\n"); 1323 } 1324 return rv; 1325 } 1326 1327 void siw_free_qp(struct kref *ref) 1328 { 1329 struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref); 1330 struct siw_device *sdev = qp->sdev; 1331 unsigned long flags; 1332 1333 if (qp->cep) 1334 siw_cep_put(qp->cep); 1335 1336 found = xa_erase(&sdev->qp_xa, qp_id(qp)); 1337 WARN_ON(found != qp); 1338 spin_lock_irqsave(&sdev->lock, flags); 1339 list_del(&qp->devq); 1340 spin_unlock_irqrestore(&sdev->lock, flags); 1341 1342 vfree(qp->sendq); 1343 vfree(qp->recvq); 1344 vfree(qp->irq); 1345 vfree(qp->orq); 1346 1347 siw_put_tx_cpu(qp->tx_cpu); 1348 complete(&qp->qp_free); 1349 atomic_dec(&sdev->num_qp); 1350 } 1351