1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2 3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 4 /* Copyright (c) 2008-2019, IBM Corporation */ 5 6 #include <linux/errno.h> 7 #include <linux/types.h> 8 #include <linux/net.h> 9 #include <linux/scatterlist.h> 10 #include <linux/llist.h> 11 #include <asm/barrier.h> 12 #include <net/tcp.h> 13 14 #include "siw.h" 15 #include "siw_verbs.h" 16 #include "siw_mem.h" 17 18 static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = { 19 [SIW_QP_STATE_IDLE] = "IDLE", 20 [SIW_QP_STATE_RTR] = "RTR", 21 [SIW_QP_STATE_RTS] = "RTS", 22 [SIW_QP_STATE_CLOSING] = "CLOSING", 23 [SIW_QP_STATE_TERMINATE] = "TERMINATE", 24 [SIW_QP_STATE_ERROR] = "ERROR" 25 }; 26 27 /* 28 * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a 29 * per-RDMAP message basis. Please keep order of initializer. All MPA len 30 * is initialized to minimum packet size. 31 */ 32 struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = { 33 { /* RDMAP_RDMA_WRITE */ 34 .hdr_len = sizeof(struct iwarp_rdma_write), 35 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2), 36 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST | 37 cpu_to_be16(DDP_VERSION << 8) | 38 cpu_to_be16(RDMAP_VERSION << 6) | 39 cpu_to_be16(RDMAP_RDMA_WRITE), 40 .rx_data = siw_proc_write }, 41 { /* RDMAP_RDMA_READ_REQ */ 42 .hdr_len = sizeof(struct iwarp_rdma_rreq), 43 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2), 44 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 45 cpu_to_be16(RDMAP_VERSION << 6) | 46 cpu_to_be16(RDMAP_RDMA_READ_REQ), 47 .rx_data = siw_proc_rreq }, 48 { /* RDMAP_RDMA_READ_RESP */ 49 .hdr_len = sizeof(struct iwarp_rdma_rresp), 50 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2), 51 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST | 52 cpu_to_be16(DDP_VERSION << 8) | 53 cpu_to_be16(RDMAP_VERSION << 6) | 54 cpu_to_be16(RDMAP_RDMA_READ_RESP), 55 .rx_data = siw_proc_rresp }, 56 { /* RDMAP_SEND */ 57 .hdr_len = sizeof(struct iwarp_send), 58 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2), 59 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 60 cpu_to_be16(RDMAP_VERSION << 6) | 61 cpu_to_be16(RDMAP_SEND), 62 .rx_data = siw_proc_send }, 63 { /* RDMAP_SEND_INVAL */ 64 .hdr_len = sizeof(struct iwarp_send_inv), 65 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2), 66 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 67 cpu_to_be16(RDMAP_VERSION << 6) | 68 cpu_to_be16(RDMAP_SEND_INVAL), 69 .rx_data = siw_proc_send }, 70 { /* RDMAP_SEND_SE */ 71 .hdr_len = sizeof(struct iwarp_send), 72 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2), 73 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 74 cpu_to_be16(RDMAP_VERSION << 6) | 75 cpu_to_be16(RDMAP_SEND_SE), 76 .rx_data = siw_proc_send }, 77 { /* RDMAP_SEND_SE_INVAL */ 78 .hdr_len = sizeof(struct iwarp_send_inv), 79 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2), 80 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 81 cpu_to_be16(RDMAP_VERSION << 6) | 82 cpu_to_be16(RDMAP_SEND_SE_INVAL), 83 .rx_data = siw_proc_send }, 84 { /* RDMAP_TERMINATE */ 85 .hdr_len = sizeof(struct iwarp_terminate), 86 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2), 87 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 88 cpu_to_be16(RDMAP_VERSION << 6) | 89 cpu_to_be16(RDMAP_TERMINATE), 90 .rx_data = siw_proc_terminate } 91 }; 92 93 void siw_qp_llp_data_ready(struct sock *sk) 94 { 95 struct siw_qp *qp; 96 97 read_lock(&sk->sk_callback_lock); 98 99 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk))) 100 goto done; 101 102 qp = sk_to_qp(sk); 103 104 if (likely(!qp->rx_stream.rx_suspend && 105 down_read_trylock(&qp->state_lock))) { 106 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 }; 107 108 if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) 109 /* 110 * Implements data receive operation during 111 * socket callback. TCP gracefully catches 112 * the case where there is nothing to receive 113 * (not calling siw_tcp_rx_data() then). 114 */ 115 tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data); 116 117 up_read(&qp->state_lock); 118 } else { 119 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n", 120 qp->rx_stream.rx_suspend); 121 } 122 done: 123 read_unlock(&sk->sk_callback_lock); 124 } 125 126 void siw_qp_llp_close(struct siw_qp *qp) 127 { 128 siw_dbg_qp(qp, "enter llp close, state = %s\n", 129 siw_qp_state_to_string[qp->attrs.state]); 130 131 down_write(&qp->state_lock); 132 133 qp->rx_stream.rx_suspend = 1; 134 qp->tx_ctx.tx_suspend = 1; 135 qp->attrs.sk = NULL; 136 137 switch (qp->attrs.state) { 138 case SIW_QP_STATE_RTS: 139 case SIW_QP_STATE_RTR: 140 case SIW_QP_STATE_IDLE: 141 case SIW_QP_STATE_TERMINATE: 142 qp->attrs.state = SIW_QP_STATE_ERROR; 143 break; 144 /* 145 * SIW_QP_STATE_CLOSING: 146 * 147 * This is a forced close. shall the QP be moved to 148 * ERROR or IDLE ? 149 */ 150 case SIW_QP_STATE_CLOSING: 151 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) 152 qp->attrs.state = SIW_QP_STATE_ERROR; 153 else 154 qp->attrs.state = SIW_QP_STATE_IDLE; 155 break; 156 157 default: 158 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n", 159 siw_qp_state_to_string[qp->attrs.state]); 160 break; 161 } 162 siw_sq_flush(qp); 163 siw_rq_flush(qp); 164 165 /* 166 * Dereference closing CEP 167 */ 168 if (qp->cep) { 169 siw_cep_put(qp->cep); 170 qp->cep = NULL; 171 } 172 173 up_write(&qp->state_lock); 174 175 siw_dbg_qp(qp, "llp close exit: state %s\n", 176 siw_qp_state_to_string[qp->attrs.state]); 177 } 178 179 /* 180 * socket callback routine informing about newly available send space. 181 * Function schedules SQ work for processing SQ items. 182 */ 183 void siw_qp_llp_write_space(struct sock *sk) 184 { 185 struct siw_cep *cep; 186 187 read_lock(&sk->sk_callback_lock); 188 189 cep = sk_to_cep(sk); 190 if (cep) { 191 cep->sk_write_space(sk); 192 193 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) 194 (void)siw_sq_start(cep->qp); 195 } 196 197 read_unlock(&sk->sk_callback_lock); 198 } 199 200 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size) 201 { 202 if (irq_size) { 203 irq_size = roundup_pow_of_two(irq_size); 204 qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe)); 205 if (!qp->irq) { 206 qp->attrs.irq_size = 0; 207 return -ENOMEM; 208 } 209 } 210 if (orq_size) { 211 orq_size = roundup_pow_of_two(orq_size); 212 qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe)); 213 if (!qp->orq) { 214 qp->attrs.orq_size = 0; 215 qp->attrs.irq_size = 0; 216 vfree(qp->irq); 217 return -ENOMEM; 218 } 219 } 220 qp->attrs.irq_size = irq_size; 221 qp->attrs.orq_size = orq_size; 222 siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size); 223 return 0; 224 } 225 226 static int siw_qp_enable_crc(struct siw_qp *qp) 227 { 228 struct siw_rx_stream *c_rx = &qp->rx_stream; 229 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; 230 int size; 231 232 if (siw_crypto_shash == NULL) 233 return -ENOENT; 234 235 size = crypto_shash_descsize(siw_crypto_shash) + 236 sizeof(struct shash_desc); 237 238 c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); 239 c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); 240 if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) { 241 kfree(c_tx->mpa_crc_hd); 242 kfree(c_rx->mpa_crc_hd); 243 c_tx->mpa_crc_hd = NULL; 244 c_rx->mpa_crc_hd = NULL; 245 return -ENOMEM; 246 } 247 c_tx->mpa_crc_hd->tfm = siw_crypto_shash; 248 c_rx->mpa_crc_hd->tfm = siw_crypto_shash; 249 250 return 0; 251 } 252 253 /* 254 * Send a non signalled READ or WRITE to peer side as negotiated 255 * with MPAv2 P2P setup protocol. The work request is only created 256 * as a current active WR and does not consume Send Queue space. 257 * 258 * Caller must hold QP state lock. 259 */ 260 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl) 261 { 262 struct siw_wqe *wqe = tx_wqe(qp); 263 unsigned long flags; 264 int rv = 0; 265 266 spin_lock_irqsave(&qp->sq_lock, flags); 267 268 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { 269 spin_unlock_irqrestore(&qp->sq_lock, flags); 270 return -EIO; 271 } 272 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 273 274 wqe->wr_status = SIW_WR_QUEUED; 275 wqe->sqe.flags = 0; 276 wqe->sqe.num_sge = 1; 277 wqe->sqe.sge[0].length = 0; 278 wqe->sqe.sge[0].laddr = 0; 279 wqe->sqe.sge[0].lkey = 0; 280 /* 281 * While it must not be checked for inbound zero length 282 * READ/WRITE, some HW may treat STag 0 special. 283 */ 284 wqe->sqe.rkey = 1; 285 wqe->sqe.raddr = 0; 286 wqe->processed = 0; 287 288 if (ctrl & MPA_V2_RDMA_WRITE_RTR) 289 wqe->sqe.opcode = SIW_OP_WRITE; 290 else if (ctrl & MPA_V2_RDMA_READ_RTR) { 291 struct siw_sqe *rreq = NULL; 292 293 wqe->sqe.opcode = SIW_OP_READ; 294 295 spin_lock(&qp->orq_lock); 296 297 if (qp->attrs.orq_size) 298 rreq = orq_get_free(qp); 299 if (rreq) { 300 siw_read_to_orq(rreq, &wqe->sqe); 301 qp->orq_put++; 302 } else 303 rv = -EIO; 304 305 spin_unlock(&qp->orq_lock); 306 } else 307 rv = -EINVAL; 308 309 if (rv) 310 wqe->wr_status = SIW_WR_IDLE; 311 312 spin_unlock_irqrestore(&qp->sq_lock, flags); 313 314 if (!rv) 315 rv = siw_sq_start(qp); 316 317 return rv; 318 } 319 320 /* 321 * Map memory access error to DDP tagged error 322 */ 323 enum ddp_ecode siw_tagged_error(enum siw_access_state state) 324 { 325 switch (state) { 326 case E_STAG_INVALID: 327 return DDP_ECODE_T_INVALID_STAG; 328 case E_BASE_BOUNDS: 329 return DDP_ECODE_T_BASE_BOUNDS; 330 case E_PD_MISMATCH: 331 return DDP_ECODE_T_STAG_NOT_ASSOC; 332 case E_ACCESS_PERM: 333 /* 334 * RFC 5041 (DDP) lacks an ecode for insufficient access 335 * permissions. 'Invalid STag' seem to be the closest 336 * match though. 337 */ 338 return DDP_ECODE_T_INVALID_STAG; 339 default: 340 WARN_ON(1); 341 return DDP_ECODE_T_INVALID_STAG; 342 } 343 } 344 345 /* 346 * Map memory access error to RDMAP protection error 347 */ 348 enum rdmap_ecode siw_rdmap_error(enum siw_access_state state) 349 { 350 switch (state) { 351 case E_STAG_INVALID: 352 return RDMAP_ECODE_INVALID_STAG; 353 case E_BASE_BOUNDS: 354 return RDMAP_ECODE_BASE_BOUNDS; 355 case E_PD_MISMATCH: 356 return RDMAP_ECODE_STAG_NOT_ASSOC; 357 case E_ACCESS_PERM: 358 return RDMAP_ECODE_ACCESS_RIGHTS; 359 default: 360 return RDMAP_ECODE_UNSPECIFIED; 361 } 362 } 363 364 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype, 365 u8 ecode, int in_tx) 366 { 367 if (!qp->term_info.valid) { 368 memset(&qp->term_info, 0, sizeof(qp->term_info)); 369 qp->term_info.layer = layer; 370 qp->term_info.etype = etype; 371 qp->term_info.ecode = ecode; 372 qp->term_info.in_tx = in_tx; 373 qp->term_info.valid = 1; 374 } 375 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n", 376 layer, etype, ecode, in_tx ? "yes" : "no"); 377 } 378 379 /* 380 * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581. 381 * Sending TERMINATE messages is best effort - such messages 382 * can only be send if the QP is still connected and it does 383 * not have another outbound message in-progress, i.e. the 384 * TERMINATE message must not interfer with an incomplete current 385 * transmit operation. 386 */ 387 void siw_send_terminate(struct siw_qp *qp) 388 { 389 struct kvec iov[3]; 390 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR }; 391 struct iwarp_terminate *term = NULL; 392 union iwarp_hdr *err_hdr = NULL; 393 struct socket *s = qp->attrs.sk; 394 struct siw_rx_stream *srx = &qp->rx_stream; 395 union iwarp_hdr *rx_hdr = &srx->hdr; 396 u32 crc = 0; 397 int num_frags, len_terminate, rv; 398 399 if (!qp->term_info.valid) 400 return; 401 402 qp->term_info.valid = 0; 403 404 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { 405 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n", 406 tx_type(tx_wqe(qp))); 407 return; 408 } 409 if (!s && qp->cep) 410 /* QP not yet in RTS. Take socket from connection end point */ 411 s = qp->cep->sock; 412 413 if (!s) { 414 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n"); 415 return; 416 } 417 418 term = kzalloc(sizeof(*term), GFP_KERNEL); 419 if (!term) 420 return; 421 422 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE); 423 term->ddp_mo = 0; 424 term->ddp_msn = cpu_to_be32(1); 425 426 iov[0].iov_base = term; 427 iov[0].iov_len = sizeof(*term); 428 429 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || 430 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && 431 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { 432 err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL); 433 if (!err_hdr) { 434 kfree(term); 435 return; 436 } 437 } 438 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl, 439 sizeof(struct iwarp_ctrl)); 440 441 __rdmap_term_set_layer(term, qp->term_info.layer); 442 __rdmap_term_set_etype(term, qp->term_info.etype); 443 __rdmap_term_set_ecode(term, qp->term_info.ecode); 444 445 switch (qp->term_info.layer) { 446 case TERM_ERROR_LAYER_RDMAP: 447 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) 448 /* No additional DDP/RDMAP header to be included */ 449 break; 450 451 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { 452 /* 453 * Complete RDMAP frame will get attached, and 454 * DDP segment length is valid 455 */ 456 term->flag_m = 1; 457 term->flag_d = 1; 458 term->flag_r = 1; 459 460 if (qp->term_info.in_tx) { 461 struct iwarp_rdma_rreq *rreq; 462 struct siw_wqe *wqe = tx_wqe(qp); 463 464 /* Inbound RREQ error, detected during 465 * RRESP creation. Take state from 466 * current TX work queue element to 467 * reconstruct peers RREQ. 468 */ 469 rreq = (struct iwarp_rdma_rreq *)err_hdr; 470 471 memcpy(&rreq->ctrl, 472 &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl, 473 sizeof(struct iwarp_ctrl)); 474 475 rreq->rsvd = 0; 476 rreq->ddp_qn = 477 htonl(RDMAP_UNTAGGED_QN_RDMA_READ); 478 479 /* Provide RREQ's MSN as kept aside */ 480 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length); 481 482 rreq->ddp_mo = htonl(wqe->processed); 483 rreq->sink_stag = htonl(wqe->sqe.rkey); 484 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr); 485 rreq->read_size = htonl(wqe->sqe.sge[0].length); 486 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey); 487 rreq->source_to = 488 cpu_to_be64(wqe->sqe.sge[0].laddr); 489 490 iov[1].iov_base = rreq; 491 iov[1].iov_len = sizeof(*rreq); 492 493 rx_hdr = (union iwarp_hdr *)rreq; 494 } else { 495 /* Take RDMAP/DDP information from 496 * current (failed) inbound frame. 497 */ 498 iov[1].iov_base = rx_hdr; 499 500 if (__rdmap_get_opcode(&rx_hdr->ctrl) == 501 RDMAP_RDMA_READ_REQ) 502 iov[1].iov_len = 503 sizeof(struct iwarp_rdma_rreq); 504 else /* SEND type */ 505 iov[1].iov_len = 506 sizeof(struct iwarp_send); 507 } 508 } else { 509 /* Do not report DDP hdr information if packet 510 * layout is unknown 511 */ 512 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || 513 (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) 514 break; 515 516 iov[1].iov_base = rx_hdr; 517 518 /* Only DDP frame will get attached */ 519 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) 520 iov[1].iov_len = 521 sizeof(struct iwarp_rdma_write); 522 else 523 iov[1].iov_len = sizeof(struct iwarp_send); 524 525 term->flag_m = 1; 526 term->flag_d = 1; 527 } 528 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len); 529 break; 530 531 case TERM_ERROR_LAYER_DDP: 532 /* Report error encountered while DDP processing. 533 * This can only happen as a result of inbound 534 * DDP processing 535 */ 536 537 /* Do not report DDP hdr information if packet 538 * layout is unknown 539 */ 540 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && 541 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || 542 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && 543 (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) 544 break; 545 546 iov[1].iov_base = rx_hdr; 547 548 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) 549 iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged); 550 else 551 iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged); 552 553 term->flag_m = 1; 554 term->flag_d = 1; 555 break; 556 557 default: 558 break; 559 } 560 if (term->flag_m || term->flag_d || term->flag_r) { 561 iov[2].iov_base = &crc; 562 iov[2].iov_len = sizeof(crc); 563 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE; 564 num_frags = 3; 565 } else { 566 iov[1].iov_base = &crc; 567 iov[1].iov_len = sizeof(crc); 568 len_terminate = sizeof(*term) + MPA_CRC_SIZE; 569 num_frags = 2; 570 } 571 572 /* Adjust DDP Segment Length parameter, if valid */ 573 if (term->flag_m) { 574 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len); 575 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl); 576 577 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE; 578 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len); 579 } 580 581 term->ctrl.mpa_len = 582 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE)); 583 if (qp->tx_ctx.mpa_crc_hd) { 584 crypto_shash_init(qp->tx_ctx.mpa_crc_hd); 585 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, 586 (u8 *)iov[0].iov_base, 587 iov[0].iov_len)) 588 goto out; 589 590 if (num_frags == 3) { 591 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, 592 (u8 *)iov[1].iov_base, 593 iov[1].iov_len)) 594 goto out; 595 } 596 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc); 597 } 598 599 rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate); 600 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n", 601 rv == len_terminate ? "success" : "failure", 602 __rdmap_term_layer(term), __rdmap_term_etype(term), 603 __rdmap_term_ecode(term), rv); 604 out: 605 kfree(term); 606 kfree(err_hdr); 607 } 608 609 /* 610 * Handle all attrs other than state 611 */ 612 static void siw_qp_modify_nonstate(struct siw_qp *qp, 613 struct siw_qp_attrs *attrs, 614 enum siw_qp_attr_mask mask) 615 { 616 if (mask & SIW_QP_ATTR_ACCESS_FLAGS) { 617 if (attrs->flags & SIW_RDMA_BIND_ENABLED) 618 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; 619 else 620 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; 621 622 if (attrs->flags & SIW_RDMA_WRITE_ENABLED) 623 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; 624 else 625 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; 626 627 if (attrs->flags & SIW_RDMA_READ_ENABLED) 628 qp->attrs.flags |= SIW_RDMA_READ_ENABLED; 629 else 630 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; 631 } 632 } 633 634 static int siw_qp_nextstate_from_idle(struct siw_qp *qp, 635 struct siw_qp_attrs *attrs, 636 enum siw_qp_attr_mask mask) 637 { 638 int rv = 0; 639 640 switch (attrs->state) { 641 case SIW_QP_STATE_RTS: 642 if (attrs->flags & SIW_MPA_CRC) { 643 rv = siw_qp_enable_crc(qp); 644 if (rv) 645 break; 646 } 647 if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) { 648 siw_dbg_qp(qp, "no socket\n"); 649 rv = -EINVAL; 650 break; 651 } 652 if (!(mask & SIW_QP_ATTR_MPA)) { 653 siw_dbg_qp(qp, "no MPA\n"); 654 rv = -EINVAL; 655 break; 656 } 657 /* 658 * Initialize iWARP TX state 659 */ 660 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; 661 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; 662 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; 663 664 /* 665 * Initialize iWARP RX state 666 */ 667 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; 668 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; 669 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; 670 671 /* 672 * init IRD free queue, caller has already checked 673 * limits. 674 */ 675 rv = siw_qp_readq_init(qp, attrs->irq_size, 676 attrs->orq_size); 677 if (rv) 678 break; 679 680 qp->attrs.sk = attrs->sk; 681 qp->attrs.state = SIW_QP_STATE_RTS; 682 683 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n", 684 attrs->flags & SIW_MPA_CRC ? "y" : "n", 685 qp->attrs.orq_size, qp->attrs.irq_size); 686 break; 687 688 case SIW_QP_STATE_ERROR: 689 siw_rq_flush(qp); 690 qp->attrs.state = SIW_QP_STATE_ERROR; 691 if (qp->cep) { 692 siw_cep_put(qp->cep); 693 qp->cep = NULL; 694 } 695 break; 696 697 default: 698 break; 699 } 700 return rv; 701 } 702 703 static int siw_qp_nextstate_from_rts(struct siw_qp *qp, 704 struct siw_qp_attrs *attrs) 705 { 706 int drop_conn = 0; 707 708 switch (attrs->state) { 709 case SIW_QP_STATE_CLOSING: 710 /* 711 * Verbs: move to IDLE if SQ and ORQ are empty. 712 * Move to ERROR otherwise. But first of all we must 713 * close the connection. So we keep CLOSING or ERROR 714 * as a transient state, schedule connection drop work 715 * and wait for the socket state change upcall to 716 * come back closed. 717 */ 718 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { 719 qp->attrs.state = SIW_QP_STATE_CLOSING; 720 } else { 721 qp->attrs.state = SIW_QP_STATE_ERROR; 722 siw_sq_flush(qp); 723 } 724 siw_rq_flush(qp); 725 726 drop_conn = 1; 727 break; 728 729 case SIW_QP_STATE_TERMINATE: 730 qp->attrs.state = SIW_QP_STATE_TERMINATE; 731 732 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, 733 RDMAP_ETYPE_CATASTROPHIC, 734 RDMAP_ECODE_UNSPECIFIED, 1); 735 drop_conn = 1; 736 break; 737 738 case SIW_QP_STATE_ERROR: 739 /* 740 * This is an emergency close. 741 * 742 * Any in progress transmit operation will get 743 * cancelled. 744 * This will likely result in a protocol failure, 745 * if a TX operation is in transit. The caller 746 * could unconditional wait to give the current 747 * operation a chance to complete. 748 * Esp., how to handle the non-empty IRQ case? 749 * The peer was asking for data transfer at a valid 750 * point in time. 751 */ 752 siw_sq_flush(qp); 753 siw_rq_flush(qp); 754 qp->attrs.state = SIW_QP_STATE_ERROR; 755 drop_conn = 1; 756 break; 757 758 default: 759 break; 760 } 761 return drop_conn; 762 } 763 764 static void siw_qp_nextstate_from_term(struct siw_qp *qp, 765 struct siw_qp_attrs *attrs) 766 { 767 switch (attrs->state) { 768 case SIW_QP_STATE_ERROR: 769 siw_rq_flush(qp); 770 qp->attrs.state = SIW_QP_STATE_ERROR; 771 772 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) 773 siw_sq_flush(qp); 774 break; 775 776 default: 777 break; 778 } 779 } 780 781 static int siw_qp_nextstate_from_close(struct siw_qp *qp, 782 struct siw_qp_attrs *attrs) 783 { 784 int rv = 0; 785 786 switch (attrs->state) { 787 case SIW_QP_STATE_IDLE: 788 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); 789 qp->attrs.state = SIW_QP_STATE_IDLE; 790 break; 791 792 case SIW_QP_STATE_CLOSING: 793 /* 794 * The LLP may already moved the QP to closing 795 * due to graceful peer close init 796 */ 797 break; 798 799 case SIW_QP_STATE_ERROR: 800 /* 801 * QP was moved to CLOSING by LLP event 802 * not yet seen by user. 803 */ 804 qp->attrs.state = SIW_QP_STATE_ERROR; 805 806 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) 807 siw_sq_flush(qp); 808 809 siw_rq_flush(qp); 810 break; 811 812 default: 813 siw_dbg_qp(qp, "state transition undefined: %s => %s\n", 814 siw_qp_state_to_string[qp->attrs.state], 815 siw_qp_state_to_string[attrs->state]); 816 817 rv = -ECONNABORTED; 818 } 819 return rv; 820 } 821 822 /* 823 * Caller must hold qp->state_lock 824 */ 825 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs, 826 enum siw_qp_attr_mask mask) 827 { 828 int drop_conn = 0, rv = 0; 829 830 if (!mask) 831 return 0; 832 833 siw_dbg_qp(qp, "state: %s => %s\n", 834 siw_qp_state_to_string[qp->attrs.state], 835 siw_qp_state_to_string[attrs->state]); 836 837 if (mask != SIW_QP_ATTR_STATE) 838 siw_qp_modify_nonstate(qp, attrs, mask); 839 840 if (!(mask & SIW_QP_ATTR_STATE)) 841 return 0; 842 843 switch (qp->attrs.state) { 844 case SIW_QP_STATE_IDLE: 845 case SIW_QP_STATE_RTR: 846 rv = siw_qp_nextstate_from_idle(qp, attrs, mask); 847 break; 848 849 case SIW_QP_STATE_RTS: 850 drop_conn = siw_qp_nextstate_from_rts(qp, attrs); 851 break; 852 853 case SIW_QP_STATE_TERMINATE: 854 siw_qp_nextstate_from_term(qp, attrs); 855 break; 856 857 case SIW_QP_STATE_CLOSING: 858 siw_qp_nextstate_from_close(qp, attrs); 859 break; 860 default: 861 break; 862 } 863 if (drop_conn) 864 siw_qp_cm_drop(qp, 0); 865 866 return rv; 867 } 868 869 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe) 870 { 871 rreq->id = sqe->id; 872 rreq->opcode = sqe->opcode; 873 rreq->sge[0].laddr = sqe->sge[0].laddr; 874 rreq->sge[0].length = sqe->sge[0].length; 875 rreq->sge[0].lkey = sqe->sge[0].lkey; 876 rreq->sge[1].lkey = sqe->sge[1].lkey; 877 rreq->flags = sqe->flags | SIW_WQE_VALID; 878 rreq->num_sge = 1; 879 } 880 881 static int siw_activate_tx_from_sq(struct siw_qp *qp) 882 { 883 struct siw_sqe *sqe; 884 struct siw_wqe *wqe = tx_wqe(qp); 885 int rv = 1; 886 887 sqe = sq_get_next(qp); 888 if (!sqe) 889 return 0; 890 891 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 892 wqe->wr_status = SIW_WR_QUEUED; 893 894 /* First copy SQE to kernel private memory */ 895 memcpy(&wqe->sqe, sqe, sizeof(*sqe)); 896 897 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) { 898 rv = -EINVAL; 899 goto out; 900 } 901 if (wqe->sqe.flags & SIW_WQE_INLINE) { 902 if (wqe->sqe.opcode != SIW_OP_SEND && 903 wqe->sqe.opcode != SIW_OP_WRITE) { 904 rv = -EINVAL; 905 goto out; 906 } 907 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) { 908 rv = -EINVAL; 909 goto out; 910 } 911 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1]; 912 wqe->sqe.sge[0].lkey = 0; 913 wqe->sqe.num_sge = 1; 914 } 915 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) { 916 /* A READ cannot be fenced */ 917 if (unlikely(wqe->sqe.opcode == SIW_OP_READ || 918 wqe->sqe.opcode == 919 SIW_OP_READ_LOCAL_INV)) { 920 siw_dbg_qp(qp, "cannot fence read\n"); 921 rv = -EINVAL; 922 goto out; 923 } 924 spin_lock(&qp->orq_lock); 925 926 if (qp->attrs.orq_size && !siw_orq_empty(qp)) { 927 qp->tx_ctx.orq_fence = 1; 928 rv = 0; 929 } 930 spin_unlock(&qp->orq_lock); 931 932 } else if (wqe->sqe.opcode == SIW_OP_READ || 933 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) { 934 struct siw_sqe *rreq; 935 936 if (unlikely(!qp->attrs.orq_size)) { 937 /* We negotiated not to send READ req's */ 938 rv = -EINVAL; 939 goto out; 940 } 941 wqe->sqe.num_sge = 1; 942 943 spin_lock(&qp->orq_lock); 944 945 rreq = orq_get_free(qp); 946 if (rreq) { 947 /* 948 * Make an immediate copy in ORQ to be ready 949 * to process loopback READ reply 950 */ 951 siw_read_to_orq(rreq, &wqe->sqe); 952 qp->orq_put++; 953 } else { 954 qp->tx_ctx.orq_fence = 1; 955 rv = 0; 956 } 957 spin_unlock(&qp->orq_lock); 958 } 959 960 /* Clear SQE, can be re-used by application */ 961 smp_store_mb(sqe->flags, 0); 962 qp->sq_get++; 963 out: 964 if (unlikely(rv < 0)) { 965 siw_dbg_qp(qp, "error %d\n", rv); 966 wqe->wr_status = SIW_WR_IDLE; 967 } 968 return rv; 969 } 970 971 /* 972 * Must be called with SQ locked. 973 * To avoid complete SQ starvation by constant inbound READ requests, 974 * the active IRQ will not be served after qp->irq_burst, if the 975 * SQ has pending work. 976 */ 977 int siw_activate_tx(struct siw_qp *qp) 978 { 979 struct siw_sqe *irqe; 980 struct siw_wqe *wqe = tx_wqe(qp); 981 982 if (!qp->attrs.irq_size) 983 return siw_activate_tx_from_sq(qp); 984 985 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; 986 987 if (!(irqe->flags & SIW_WQE_VALID)) 988 return siw_activate_tx_from_sq(qp); 989 990 /* 991 * Avoid local WQE processing starvation in case 992 * of constant inbound READ request stream 993 */ 994 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { 995 qp->irq_burst = 0; 996 return siw_activate_tx_from_sq(qp); 997 } 998 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 999 wqe->wr_status = SIW_WR_QUEUED; 1000 1001 /* start READ RESPONSE */ 1002 wqe->sqe.opcode = SIW_OP_READ_RESPONSE; 1003 wqe->sqe.flags = 0; 1004 if (irqe->num_sge) { 1005 wqe->sqe.num_sge = 1; 1006 wqe->sqe.sge[0].length = irqe->sge[0].length; 1007 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr; 1008 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey; 1009 } else { 1010 wqe->sqe.num_sge = 0; 1011 } 1012 1013 /* Retain original RREQ's message sequence number for 1014 * potential error reporting cases. 1015 */ 1016 wqe->sqe.sge[1].length = irqe->sge[1].length; 1017 1018 wqe->sqe.rkey = irqe->rkey; 1019 wqe->sqe.raddr = irqe->raddr; 1020 1021 wqe->processed = 0; 1022 qp->irq_get++; 1023 1024 /* mark current IRQ entry free */ 1025 smp_store_mb(irqe->flags, 0); 1026 1027 return 1; 1028 } 1029 1030 /* 1031 * Check if current CQ state qualifies for calling CQ completion 1032 * handler. Must be called with CQ lock held. 1033 */ 1034 static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags) 1035 { 1036 u32 cq_notify; 1037 1038 if (!cq->base_cq.comp_handler) 1039 return false; 1040 1041 /* Read application shared notification state */ 1042 cq_notify = READ_ONCE(cq->notify->flags); 1043 1044 if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) || 1045 ((cq_notify & SIW_NOTIFY_SOLICITED) && 1046 (flags & SIW_WQE_SOLICITED))) { 1047 /* 1048 * CQ notification is one-shot: Since the 1049 * current CQE causes user notification, 1050 * the CQ gets dis-aremd and must be re-aremd 1051 * by the user for a new notification. 1052 */ 1053 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT); 1054 1055 return true; 1056 } 1057 return false; 1058 } 1059 1060 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, 1061 enum siw_wc_status status) 1062 { 1063 struct siw_cq *cq = qp->scq; 1064 int rv = 0; 1065 1066 if (cq) { 1067 u32 sqe_flags = sqe->flags; 1068 struct siw_cqe *cqe; 1069 u32 idx; 1070 unsigned long flags; 1071 1072 spin_lock_irqsave(&cq->lock, flags); 1073 1074 idx = cq->cq_put % cq->num_cqe; 1075 cqe = &cq->queue[idx]; 1076 1077 if (!READ_ONCE(cqe->flags)) { 1078 bool notify; 1079 1080 cqe->id = sqe->id; 1081 cqe->opcode = sqe->opcode; 1082 cqe->status = status; 1083 cqe->imm_data = 0; 1084 cqe->bytes = bytes; 1085 1086 if (rdma_is_kernel_res(&cq->base_cq.res)) 1087 cqe->base_qp = &qp->base_qp; 1088 else 1089 cqe->qp_id = qp_id(qp); 1090 1091 /* mark CQE valid for application */ 1092 WRITE_ONCE(cqe->flags, SIW_WQE_VALID); 1093 /* recycle SQE */ 1094 smp_store_mb(sqe->flags, 0); 1095 1096 cq->cq_put++; 1097 notify = siw_cq_notify_now(cq, sqe_flags); 1098 1099 spin_unlock_irqrestore(&cq->lock, flags); 1100 1101 if (notify) { 1102 siw_dbg_cq(cq, "Call completion handler\n"); 1103 cq->base_cq.comp_handler(&cq->base_cq, 1104 cq->base_cq.cq_context); 1105 } 1106 } else { 1107 spin_unlock_irqrestore(&cq->lock, flags); 1108 rv = -ENOMEM; 1109 siw_cq_event(cq, IB_EVENT_CQ_ERR); 1110 } 1111 } else { 1112 /* recycle SQE */ 1113 smp_store_mb(sqe->flags, 0); 1114 } 1115 return rv; 1116 } 1117 1118 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, 1119 u32 inval_stag, enum siw_wc_status status) 1120 { 1121 struct siw_cq *cq = qp->rcq; 1122 int rv = 0; 1123 1124 if (cq) { 1125 struct siw_cqe *cqe; 1126 u32 idx; 1127 unsigned long flags; 1128 1129 spin_lock_irqsave(&cq->lock, flags); 1130 1131 idx = cq->cq_put % cq->num_cqe; 1132 cqe = &cq->queue[idx]; 1133 1134 if (!READ_ONCE(cqe->flags)) { 1135 bool notify; 1136 u8 cqe_flags = SIW_WQE_VALID; 1137 1138 cqe->id = rqe->id; 1139 cqe->opcode = SIW_OP_RECEIVE; 1140 cqe->status = status; 1141 cqe->imm_data = 0; 1142 cqe->bytes = bytes; 1143 1144 if (rdma_is_kernel_res(&cq->base_cq.res)) { 1145 cqe->base_qp = &qp->base_qp; 1146 if (inval_stag) { 1147 cqe_flags |= SIW_WQE_REM_INVAL; 1148 cqe->inval_stag = inval_stag; 1149 } 1150 } else { 1151 cqe->qp_id = qp_id(qp); 1152 } 1153 /* mark CQE valid for application */ 1154 WRITE_ONCE(cqe->flags, cqe_flags); 1155 /* recycle RQE */ 1156 smp_store_mb(rqe->flags, 0); 1157 1158 cq->cq_put++; 1159 notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED); 1160 1161 spin_unlock_irqrestore(&cq->lock, flags); 1162 1163 if (notify) { 1164 siw_dbg_cq(cq, "Call completion handler\n"); 1165 cq->base_cq.comp_handler(&cq->base_cq, 1166 cq->base_cq.cq_context); 1167 } 1168 } else { 1169 spin_unlock_irqrestore(&cq->lock, flags); 1170 rv = -ENOMEM; 1171 siw_cq_event(cq, IB_EVENT_CQ_ERR); 1172 } 1173 } else { 1174 /* recycle RQE */ 1175 smp_store_mb(rqe->flags, 0); 1176 } 1177 return rv; 1178 } 1179 1180 /* 1181 * siw_sq_flush() 1182 * 1183 * Flush SQ and ORRQ entries to CQ. 1184 * 1185 * Must be called with QP state write lock held. 1186 * Therefore, SQ and ORQ lock must not be taken. 1187 */ 1188 void siw_sq_flush(struct siw_qp *qp) 1189 { 1190 struct siw_sqe *sqe; 1191 struct siw_wqe *wqe = tx_wqe(qp); 1192 int async_event = 0; 1193 1194 /* 1195 * Start with completing any work currently on the ORQ 1196 */ 1197 while (qp->attrs.orq_size) { 1198 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; 1199 if (!READ_ONCE(sqe->flags)) 1200 break; 1201 1202 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1203 break; 1204 1205 WRITE_ONCE(sqe->flags, 0); 1206 qp->orq_get++; 1207 } 1208 /* 1209 * Flush an in-progress WQE if present 1210 */ 1211 if (wqe->wr_status != SIW_WR_IDLE) { 1212 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n", 1213 tx_type(wqe), wqe->wr_status); 1214 1215 siw_wqe_put_mem(wqe, tx_type(wqe)); 1216 1217 if (tx_type(wqe) != SIW_OP_READ_RESPONSE && 1218 ((tx_type(wqe) != SIW_OP_READ && 1219 tx_type(wqe) != SIW_OP_READ_LOCAL_INV) || 1220 wqe->wr_status == SIW_WR_QUEUED)) 1221 /* 1222 * An in-progress Read Request is already in 1223 * the ORQ 1224 */ 1225 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, 1226 SIW_WC_WR_FLUSH_ERR); 1227 1228 wqe->wr_status = SIW_WR_IDLE; 1229 } 1230 /* 1231 * Flush the Send Queue 1232 */ 1233 while (qp->attrs.sq_size) { 1234 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; 1235 if (!READ_ONCE(sqe->flags)) 1236 break; 1237 1238 async_event = 1; 1239 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1240 /* 1241 * Shall IB_EVENT_SQ_DRAINED be supressed if work 1242 * completion fails? 1243 */ 1244 break; 1245 1246 WRITE_ONCE(sqe->flags, 0); 1247 qp->sq_get++; 1248 } 1249 if (async_event) 1250 siw_qp_event(qp, IB_EVENT_SQ_DRAINED); 1251 } 1252 1253 /* 1254 * siw_rq_flush() 1255 * 1256 * Flush recv queue entries to CQ. Also 1257 * takes care of pending active tagged and untagged 1258 * inbound transfers, which have target memory 1259 * referenced. 1260 * 1261 * Must be called with QP state write lock held. 1262 * Therefore, RQ lock must not be taken. 1263 */ 1264 void siw_rq_flush(struct siw_qp *qp) 1265 { 1266 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; 1267 1268 /* 1269 * Flush an in-progress untagged operation if present 1270 */ 1271 if (wqe->wr_status != SIW_WR_IDLE) { 1272 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n", 1273 rx_type(wqe), wqe->wr_status); 1274 1275 siw_wqe_put_mem(wqe, rx_type(wqe)); 1276 1277 if (rx_type(wqe) == SIW_OP_RECEIVE) { 1278 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, 1279 0, SIW_WC_WR_FLUSH_ERR); 1280 } else if (rx_type(wqe) != SIW_OP_READ && 1281 rx_type(wqe) != SIW_OP_READ_RESPONSE && 1282 rx_type(wqe) != SIW_OP_WRITE) { 1283 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); 1284 } 1285 wqe->wr_status = SIW_WR_IDLE; 1286 } 1287 wqe = &qp->rx_tagged.wqe_active; 1288 1289 if (wqe->wr_status != SIW_WR_IDLE) { 1290 siw_wqe_put_mem(wqe, rx_type(wqe)); 1291 wqe->wr_status = SIW_WR_IDLE; 1292 } 1293 /* 1294 * Flush the Receive Queue 1295 */ 1296 while (qp->attrs.rq_size) { 1297 struct siw_rqe *rqe = 1298 &qp->recvq[qp->rq_get % qp->attrs.rq_size]; 1299 1300 if (!READ_ONCE(rqe->flags)) 1301 break; 1302 1303 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1304 break; 1305 1306 WRITE_ONCE(rqe->flags, 0); 1307 qp->rq_get++; 1308 } 1309 } 1310 1311 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) 1312 { 1313 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, 1314 GFP_KERNEL); 1315 1316 if (!rv) { 1317 kref_init(&qp->ref); 1318 qp->sdev = sdev; 1319 siw_dbg_qp(qp, "new QP\n"); 1320 } 1321 return rv; 1322 } 1323 1324 void siw_free_qp(struct kref *ref) 1325 { 1326 struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref); 1327 struct siw_device *sdev = qp->sdev; 1328 unsigned long flags; 1329 1330 if (qp->cep) 1331 siw_cep_put(qp->cep); 1332 1333 found = xa_erase(&sdev->qp_xa, qp_id(qp)); 1334 WARN_ON(found != qp); 1335 spin_lock_irqsave(&sdev->lock, flags); 1336 list_del(&qp->devq); 1337 spin_unlock_irqrestore(&sdev->lock, flags); 1338 1339 vfree(qp->sendq); 1340 vfree(qp->recvq); 1341 vfree(qp->irq); 1342 vfree(qp->orq); 1343 1344 siw_put_tx_cpu(qp->tx_cpu); 1345 complete(&qp->qp_free); 1346 atomic_dec(&sdev->num_qp); 1347 } 1348