1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2 3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 4 /* Copyright (c) 2008-2019, IBM Corporation */ 5 6 #include <linux/errno.h> 7 #include <linux/types.h> 8 #include <linux/net.h> 9 #include <linux/scatterlist.h> 10 #include <linux/llist.h> 11 #include <asm/barrier.h> 12 #include <net/tcp.h> 13 14 #include "siw.h" 15 #include "siw_verbs.h" 16 #include "siw_mem.h" 17 18 static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = { 19 [SIW_QP_STATE_IDLE] = "IDLE", 20 [SIW_QP_STATE_RTR] = "RTR", 21 [SIW_QP_STATE_RTS] = "RTS", 22 [SIW_QP_STATE_CLOSING] = "CLOSING", 23 [SIW_QP_STATE_TERMINATE] = "TERMINATE", 24 [SIW_QP_STATE_ERROR] = "ERROR" 25 }; 26 27 /* 28 * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a 29 * per-RDMAP message basis. Please keep order of initializer. All MPA len 30 * is initialized to minimum packet size. 31 */ 32 struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = { 33 { /* RDMAP_RDMA_WRITE */ 34 .hdr_len = sizeof(struct iwarp_rdma_write), 35 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2), 36 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST | 37 cpu_to_be16(DDP_VERSION << 8) | 38 cpu_to_be16(RDMAP_VERSION << 6) | 39 cpu_to_be16(RDMAP_RDMA_WRITE), 40 .rx_data = siw_proc_write }, 41 { /* RDMAP_RDMA_READ_REQ */ 42 .hdr_len = sizeof(struct iwarp_rdma_rreq), 43 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2), 44 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 45 cpu_to_be16(RDMAP_VERSION << 6) | 46 cpu_to_be16(RDMAP_RDMA_READ_REQ), 47 .rx_data = siw_proc_rreq }, 48 { /* RDMAP_RDMA_READ_RESP */ 49 .hdr_len = sizeof(struct iwarp_rdma_rresp), 50 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2), 51 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST | 52 cpu_to_be16(DDP_VERSION << 8) | 53 cpu_to_be16(RDMAP_VERSION << 6) | 54 cpu_to_be16(RDMAP_RDMA_READ_RESP), 55 .rx_data = siw_proc_rresp }, 56 { /* RDMAP_SEND */ 57 .hdr_len = sizeof(struct iwarp_send), 58 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2), 59 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 60 cpu_to_be16(RDMAP_VERSION << 6) | 61 cpu_to_be16(RDMAP_SEND), 62 .rx_data = siw_proc_send }, 63 { /* RDMAP_SEND_INVAL */ 64 .hdr_len = sizeof(struct iwarp_send_inv), 65 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2), 66 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 67 cpu_to_be16(RDMAP_VERSION << 6) | 68 cpu_to_be16(RDMAP_SEND_INVAL), 69 .rx_data = siw_proc_send }, 70 { /* RDMAP_SEND_SE */ 71 .hdr_len = sizeof(struct iwarp_send), 72 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2), 73 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 74 cpu_to_be16(RDMAP_VERSION << 6) | 75 cpu_to_be16(RDMAP_SEND_SE), 76 .rx_data = siw_proc_send }, 77 { /* RDMAP_SEND_SE_INVAL */ 78 .hdr_len = sizeof(struct iwarp_send_inv), 79 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2), 80 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 81 cpu_to_be16(RDMAP_VERSION << 6) | 82 cpu_to_be16(RDMAP_SEND_SE_INVAL), 83 .rx_data = siw_proc_send }, 84 { /* RDMAP_TERMINATE */ 85 .hdr_len = sizeof(struct iwarp_terminate), 86 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2), 87 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) | 88 cpu_to_be16(RDMAP_VERSION << 6) | 89 cpu_to_be16(RDMAP_TERMINATE), 90 .rx_data = siw_proc_terminate } 91 }; 92 93 void siw_qp_llp_data_ready(struct sock *sk) 94 { 95 struct siw_qp *qp; 96 97 read_lock(&sk->sk_callback_lock); 98 99 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk))) 100 goto done; 101 102 qp = sk_to_qp(sk); 103 104 if (likely(!qp->rx_stream.rx_suspend && 105 down_read_trylock(&qp->state_lock))) { 106 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 }; 107 108 if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) 109 /* 110 * Implements data receive operation during 111 * socket callback. TCP gracefully catches 112 * the case where there is nothing to receive 113 * (not calling siw_tcp_rx_data() then). 114 */ 115 tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data); 116 117 up_read(&qp->state_lock); 118 } else { 119 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n", 120 qp->rx_stream.rx_suspend); 121 } 122 done: 123 read_unlock(&sk->sk_callback_lock); 124 } 125 126 void siw_qp_llp_close(struct siw_qp *qp) 127 { 128 siw_dbg_qp(qp, "enter llp close, state = %s\n", 129 siw_qp_state_to_string[qp->attrs.state]); 130 131 down_write(&qp->state_lock); 132 133 qp->rx_stream.rx_suspend = 1; 134 qp->tx_ctx.tx_suspend = 1; 135 qp->attrs.sk = NULL; 136 137 switch (qp->attrs.state) { 138 case SIW_QP_STATE_RTS: 139 case SIW_QP_STATE_RTR: 140 case SIW_QP_STATE_IDLE: 141 case SIW_QP_STATE_TERMINATE: 142 qp->attrs.state = SIW_QP_STATE_ERROR; 143 break; 144 /* 145 * SIW_QP_STATE_CLOSING: 146 * 147 * This is a forced close. shall the QP be moved to 148 * ERROR or IDLE ? 149 */ 150 case SIW_QP_STATE_CLOSING: 151 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) 152 qp->attrs.state = SIW_QP_STATE_ERROR; 153 else 154 qp->attrs.state = SIW_QP_STATE_IDLE; 155 break; 156 157 default: 158 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n", 159 siw_qp_state_to_string[qp->attrs.state]); 160 break; 161 } 162 siw_sq_flush(qp); 163 siw_rq_flush(qp); 164 165 /* 166 * Dereference closing CEP 167 */ 168 if (qp->cep) { 169 siw_cep_put(qp->cep); 170 qp->cep = NULL; 171 } 172 173 up_write(&qp->state_lock); 174 175 siw_dbg_qp(qp, "llp close exit: state %s\n", 176 siw_qp_state_to_string[qp->attrs.state]); 177 } 178 179 /* 180 * socket callback routine informing about newly available send space. 181 * Function schedules SQ work for processing SQ items. 182 */ 183 void siw_qp_llp_write_space(struct sock *sk) 184 { 185 struct siw_cep *cep; 186 187 read_lock(&sk->sk_callback_lock); 188 189 cep = sk_to_cep(sk); 190 if (cep) { 191 cep->sk_write_space(sk); 192 193 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) 194 (void)siw_sq_start(cep->qp); 195 } 196 197 read_unlock(&sk->sk_callback_lock); 198 } 199 200 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size) 201 { 202 irq_size = roundup_pow_of_two(irq_size); 203 orq_size = roundup_pow_of_two(orq_size); 204 205 qp->attrs.irq_size = irq_size; 206 qp->attrs.orq_size = orq_size; 207 208 qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe)); 209 if (!qp->irq) { 210 siw_dbg_qp(qp, "irq malloc for %d failed\n", irq_size); 211 qp->attrs.irq_size = 0; 212 return -ENOMEM; 213 } 214 qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe)); 215 if (!qp->orq) { 216 siw_dbg_qp(qp, "orq malloc for %d failed\n", orq_size); 217 qp->attrs.orq_size = 0; 218 qp->attrs.irq_size = 0; 219 vfree(qp->irq); 220 return -ENOMEM; 221 } 222 siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size); 223 return 0; 224 } 225 226 static int siw_qp_enable_crc(struct siw_qp *qp) 227 { 228 struct siw_rx_stream *c_rx = &qp->rx_stream; 229 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; 230 int size; 231 232 if (siw_crypto_shash == NULL) 233 return -ENOENT; 234 235 size = crypto_shash_descsize(siw_crypto_shash) + 236 sizeof(struct shash_desc); 237 238 c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); 239 c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL); 240 if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) { 241 kfree(c_tx->mpa_crc_hd); 242 kfree(c_rx->mpa_crc_hd); 243 c_tx->mpa_crc_hd = NULL; 244 c_rx->mpa_crc_hd = NULL; 245 return -ENOMEM; 246 } 247 c_tx->mpa_crc_hd->tfm = siw_crypto_shash; 248 c_rx->mpa_crc_hd->tfm = siw_crypto_shash; 249 250 return 0; 251 } 252 253 /* 254 * Send a non signalled READ or WRITE to peer side as negotiated 255 * with MPAv2 P2P setup protocol. The work request is only created 256 * as a current active WR and does not consume Send Queue space. 257 * 258 * Caller must hold QP state lock. 259 */ 260 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl) 261 { 262 struct siw_wqe *wqe = tx_wqe(qp); 263 unsigned long flags; 264 int rv = 0; 265 266 spin_lock_irqsave(&qp->sq_lock, flags); 267 268 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { 269 spin_unlock_irqrestore(&qp->sq_lock, flags); 270 return -EIO; 271 } 272 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 273 274 wqe->wr_status = SIW_WR_QUEUED; 275 wqe->sqe.flags = 0; 276 wqe->sqe.num_sge = 1; 277 wqe->sqe.sge[0].length = 0; 278 wqe->sqe.sge[0].laddr = 0; 279 wqe->sqe.sge[0].lkey = 0; 280 /* 281 * While it must not be checked for inbound zero length 282 * READ/WRITE, some HW may treat STag 0 special. 283 */ 284 wqe->sqe.rkey = 1; 285 wqe->sqe.raddr = 0; 286 wqe->processed = 0; 287 288 if (ctrl & MPA_V2_RDMA_WRITE_RTR) 289 wqe->sqe.opcode = SIW_OP_WRITE; 290 else if (ctrl & MPA_V2_RDMA_READ_RTR) { 291 struct siw_sqe *rreq; 292 293 wqe->sqe.opcode = SIW_OP_READ; 294 295 spin_lock(&qp->orq_lock); 296 297 rreq = orq_get_free(qp); 298 if (rreq) { 299 siw_read_to_orq(rreq, &wqe->sqe); 300 qp->orq_put++; 301 } else 302 rv = -EIO; 303 304 spin_unlock(&qp->orq_lock); 305 } else 306 rv = -EINVAL; 307 308 if (rv) 309 wqe->wr_status = SIW_WR_IDLE; 310 311 spin_unlock_irqrestore(&qp->sq_lock, flags); 312 313 if (!rv) 314 rv = siw_sq_start(qp); 315 316 return rv; 317 } 318 319 /* 320 * Map memory access error to DDP tagged error 321 */ 322 enum ddp_ecode siw_tagged_error(enum siw_access_state state) 323 { 324 switch (state) { 325 case E_STAG_INVALID: 326 return DDP_ECODE_T_INVALID_STAG; 327 case E_BASE_BOUNDS: 328 return DDP_ECODE_T_BASE_BOUNDS; 329 case E_PD_MISMATCH: 330 return DDP_ECODE_T_STAG_NOT_ASSOC; 331 case E_ACCESS_PERM: 332 /* 333 * RFC 5041 (DDP) lacks an ecode for insufficient access 334 * permissions. 'Invalid STag' seem to be the closest 335 * match though. 336 */ 337 return DDP_ECODE_T_INVALID_STAG; 338 default: 339 WARN_ON(1); 340 return DDP_ECODE_T_INVALID_STAG; 341 } 342 } 343 344 /* 345 * Map memory access error to RDMAP protection error 346 */ 347 enum rdmap_ecode siw_rdmap_error(enum siw_access_state state) 348 { 349 switch (state) { 350 case E_STAG_INVALID: 351 return RDMAP_ECODE_INVALID_STAG; 352 case E_BASE_BOUNDS: 353 return RDMAP_ECODE_BASE_BOUNDS; 354 case E_PD_MISMATCH: 355 return RDMAP_ECODE_STAG_NOT_ASSOC; 356 case E_ACCESS_PERM: 357 return RDMAP_ECODE_ACCESS_RIGHTS; 358 default: 359 return RDMAP_ECODE_UNSPECIFIED; 360 } 361 } 362 363 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype, 364 u8 ecode, int in_tx) 365 { 366 if (!qp->term_info.valid) { 367 memset(&qp->term_info, 0, sizeof(qp->term_info)); 368 qp->term_info.layer = layer; 369 qp->term_info.etype = etype; 370 qp->term_info.ecode = ecode; 371 qp->term_info.in_tx = in_tx; 372 qp->term_info.valid = 1; 373 } 374 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n", 375 layer, etype, ecode, in_tx ? "yes" : "no"); 376 } 377 378 /* 379 * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581. 380 * Sending TERMINATE messages is best effort - such messages 381 * can only be send if the QP is still connected and it does 382 * not have another outbound message in-progress, i.e. the 383 * TERMINATE message must not interfer with an incomplete current 384 * transmit operation. 385 */ 386 void siw_send_terminate(struct siw_qp *qp) 387 { 388 struct kvec iov[3]; 389 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR }; 390 struct iwarp_terminate *term = NULL; 391 union iwarp_hdr *err_hdr = NULL; 392 struct socket *s = qp->attrs.sk; 393 struct siw_rx_stream *srx = &qp->rx_stream; 394 union iwarp_hdr *rx_hdr = &srx->hdr; 395 u32 crc = 0; 396 int num_frags, len_terminate, rv; 397 398 if (!qp->term_info.valid) 399 return; 400 401 qp->term_info.valid = 0; 402 403 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { 404 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n", 405 tx_type(tx_wqe(qp))); 406 return; 407 } 408 if (!s && qp->cep) 409 /* QP not yet in RTS. Take socket from connection end point */ 410 s = qp->cep->sock; 411 412 if (!s) { 413 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n"); 414 return; 415 } 416 417 term = kzalloc(sizeof(*term), GFP_KERNEL); 418 if (!term) 419 return; 420 421 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE); 422 term->ddp_mo = 0; 423 term->ddp_msn = cpu_to_be32(1); 424 425 iov[0].iov_base = term; 426 iov[0].iov_len = sizeof(*term); 427 428 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || 429 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && 430 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { 431 err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL); 432 if (!err_hdr) { 433 kfree(term); 434 return; 435 } 436 } 437 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl, 438 sizeof(struct iwarp_ctrl)); 439 440 __rdmap_term_set_layer(term, qp->term_info.layer); 441 __rdmap_term_set_etype(term, qp->term_info.etype); 442 __rdmap_term_set_ecode(term, qp->term_info.ecode); 443 444 switch (qp->term_info.layer) { 445 case TERM_ERROR_LAYER_RDMAP: 446 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) 447 /* No additional DDP/RDMAP header to be included */ 448 break; 449 450 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { 451 /* 452 * Complete RDMAP frame will get attached, and 453 * DDP segment length is valid 454 */ 455 term->flag_m = 1; 456 term->flag_d = 1; 457 term->flag_r = 1; 458 459 if (qp->term_info.in_tx) { 460 struct iwarp_rdma_rreq *rreq; 461 struct siw_wqe *wqe = tx_wqe(qp); 462 463 /* Inbound RREQ error, detected during 464 * RRESP creation. Take state from 465 * current TX work queue element to 466 * reconstruct peers RREQ. 467 */ 468 rreq = (struct iwarp_rdma_rreq *)err_hdr; 469 470 memcpy(&rreq->ctrl, 471 &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl, 472 sizeof(struct iwarp_ctrl)); 473 474 rreq->rsvd = 0; 475 rreq->ddp_qn = 476 htonl(RDMAP_UNTAGGED_QN_RDMA_READ); 477 478 /* Provide RREQ's MSN as kept aside */ 479 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length); 480 481 rreq->ddp_mo = htonl(wqe->processed); 482 rreq->sink_stag = htonl(wqe->sqe.rkey); 483 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr); 484 rreq->read_size = htonl(wqe->sqe.sge[0].length); 485 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey); 486 rreq->source_to = 487 cpu_to_be64(wqe->sqe.sge[0].laddr); 488 489 iov[1].iov_base = rreq; 490 iov[1].iov_len = sizeof(*rreq); 491 492 rx_hdr = (union iwarp_hdr *)rreq; 493 } else { 494 /* Take RDMAP/DDP information from 495 * current (failed) inbound frame. 496 */ 497 iov[1].iov_base = rx_hdr; 498 499 if (__rdmap_get_opcode(&rx_hdr->ctrl) == 500 RDMAP_RDMA_READ_REQ) 501 iov[1].iov_len = 502 sizeof(struct iwarp_rdma_rreq); 503 else /* SEND type */ 504 iov[1].iov_len = 505 sizeof(struct iwarp_send); 506 } 507 } else { 508 /* Do not report DDP hdr information if packet 509 * layout is unknown 510 */ 511 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || 512 (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) 513 break; 514 515 iov[1].iov_base = rx_hdr; 516 517 /* Only DDP frame will get attached */ 518 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) 519 iov[1].iov_len = 520 sizeof(struct iwarp_rdma_write); 521 else 522 iov[1].iov_len = sizeof(struct iwarp_send); 523 524 term->flag_m = 1; 525 term->flag_d = 1; 526 } 527 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len); 528 break; 529 530 case TERM_ERROR_LAYER_DDP: 531 /* Report error encountered while DDP processing. 532 * This can only happen as a result of inbound 533 * DDP processing 534 */ 535 536 /* Do not report DDP hdr information if packet 537 * layout is unknown 538 */ 539 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && 540 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || 541 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && 542 (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) 543 break; 544 545 iov[1].iov_base = rx_hdr; 546 547 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) 548 iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged); 549 else 550 iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged); 551 552 term->flag_m = 1; 553 term->flag_d = 1; 554 break; 555 556 default: 557 break; 558 } 559 if (term->flag_m || term->flag_d || term->flag_r) { 560 iov[2].iov_base = &crc; 561 iov[2].iov_len = sizeof(crc); 562 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE; 563 num_frags = 3; 564 } else { 565 iov[1].iov_base = &crc; 566 iov[1].iov_len = sizeof(crc); 567 len_terminate = sizeof(*term) + MPA_CRC_SIZE; 568 num_frags = 2; 569 } 570 571 /* Adjust DDP Segment Length parameter, if valid */ 572 if (term->flag_m) { 573 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len); 574 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl); 575 576 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE; 577 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len); 578 } 579 580 term->ctrl.mpa_len = 581 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE)); 582 if (qp->tx_ctx.mpa_crc_hd) { 583 crypto_shash_init(qp->tx_ctx.mpa_crc_hd); 584 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, 585 (u8 *)iov[0].iov_base, 586 iov[0].iov_len)) 587 goto out; 588 589 if (num_frags == 3) { 590 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd, 591 (u8 *)iov[1].iov_base, 592 iov[1].iov_len)) 593 goto out; 594 } 595 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc); 596 } 597 598 rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate); 599 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n", 600 rv == len_terminate ? "success" : "failure", 601 __rdmap_term_layer(term), __rdmap_term_etype(term), 602 __rdmap_term_ecode(term), rv); 603 out: 604 kfree(term); 605 kfree(err_hdr); 606 } 607 608 /* 609 * Handle all attrs other than state 610 */ 611 static void siw_qp_modify_nonstate(struct siw_qp *qp, 612 struct siw_qp_attrs *attrs, 613 enum siw_qp_attr_mask mask) 614 { 615 if (mask & SIW_QP_ATTR_ACCESS_FLAGS) { 616 if (attrs->flags & SIW_RDMA_BIND_ENABLED) 617 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; 618 else 619 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; 620 621 if (attrs->flags & SIW_RDMA_WRITE_ENABLED) 622 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; 623 else 624 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; 625 626 if (attrs->flags & SIW_RDMA_READ_ENABLED) 627 qp->attrs.flags |= SIW_RDMA_READ_ENABLED; 628 else 629 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; 630 } 631 } 632 633 static int siw_qp_nextstate_from_idle(struct siw_qp *qp, 634 struct siw_qp_attrs *attrs, 635 enum siw_qp_attr_mask mask) 636 { 637 int rv = 0; 638 639 switch (attrs->state) { 640 case SIW_QP_STATE_RTS: 641 if (attrs->flags & SIW_MPA_CRC) { 642 rv = siw_qp_enable_crc(qp); 643 if (rv) 644 break; 645 } 646 if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) { 647 siw_dbg_qp(qp, "no socket\n"); 648 rv = -EINVAL; 649 break; 650 } 651 if (!(mask & SIW_QP_ATTR_MPA)) { 652 siw_dbg_qp(qp, "no MPA\n"); 653 rv = -EINVAL; 654 break; 655 } 656 /* 657 * Initialize iWARP TX state 658 */ 659 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; 660 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; 661 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; 662 663 /* 664 * Initialize iWARP RX state 665 */ 666 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; 667 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; 668 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; 669 670 /* 671 * init IRD free queue, caller has already checked 672 * limits. 673 */ 674 rv = siw_qp_readq_init(qp, attrs->irq_size, 675 attrs->orq_size); 676 if (rv) 677 break; 678 679 qp->attrs.sk = attrs->sk; 680 qp->attrs.state = SIW_QP_STATE_RTS; 681 682 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n", 683 attrs->flags & SIW_MPA_CRC ? "y" : "n", 684 qp->attrs.orq_size, qp->attrs.irq_size); 685 break; 686 687 case SIW_QP_STATE_ERROR: 688 siw_rq_flush(qp); 689 qp->attrs.state = SIW_QP_STATE_ERROR; 690 if (qp->cep) { 691 siw_cep_put(qp->cep); 692 qp->cep = NULL; 693 } 694 break; 695 696 default: 697 break; 698 } 699 return rv; 700 } 701 702 static int siw_qp_nextstate_from_rts(struct siw_qp *qp, 703 struct siw_qp_attrs *attrs) 704 { 705 int drop_conn = 0; 706 707 switch (attrs->state) { 708 case SIW_QP_STATE_CLOSING: 709 /* 710 * Verbs: move to IDLE if SQ and ORQ are empty. 711 * Move to ERROR otherwise. But first of all we must 712 * close the connection. So we keep CLOSING or ERROR 713 * as a transient state, schedule connection drop work 714 * and wait for the socket state change upcall to 715 * come back closed. 716 */ 717 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { 718 qp->attrs.state = SIW_QP_STATE_CLOSING; 719 } else { 720 qp->attrs.state = SIW_QP_STATE_ERROR; 721 siw_sq_flush(qp); 722 } 723 siw_rq_flush(qp); 724 725 drop_conn = 1; 726 break; 727 728 case SIW_QP_STATE_TERMINATE: 729 qp->attrs.state = SIW_QP_STATE_TERMINATE; 730 731 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, 732 RDMAP_ETYPE_CATASTROPHIC, 733 RDMAP_ECODE_UNSPECIFIED, 1); 734 drop_conn = 1; 735 break; 736 737 case SIW_QP_STATE_ERROR: 738 /* 739 * This is an emergency close. 740 * 741 * Any in progress transmit operation will get 742 * cancelled. 743 * This will likely result in a protocol failure, 744 * if a TX operation is in transit. The caller 745 * could unconditional wait to give the current 746 * operation a chance to complete. 747 * Esp., how to handle the non-empty IRQ case? 748 * The peer was asking for data transfer at a valid 749 * point in time. 750 */ 751 siw_sq_flush(qp); 752 siw_rq_flush(qp); 753 qp->attrs.state = SIW_QP_STATE_ERROR; 754 drop_conn = 1; 755 break; 756 757 default: 758 break; 759 } 760 return drop_conn; 761 } 762 763 static void siw_qp_nextstate_from_term(struct siw_qp *qp, 764 struct siw_qp_attrs *attrs) 765 { 766 switch (attrs->state) { 767 case SIW_QP_STATE_ERROR: 768 siw_rq_flush(qp); 769 qp->attrs.state = SIW_QP_STATE_ERROR; 770 771 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) 772 siw_sq_flush(qp); 773 break; 774 775 default: 776 break; 777 } 778 } 779 780 static int siw_qp_nextstate_from_close(struct siw_qp *qp, 781 struct siw_qp_attrs *attrs) 782 { 783 int rv = 0; 784 785 switch (attrs->state) { 786 case SIW_QP_STATE_IDLE: 787 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); 788 qp->attrs.state = SIW_QP_STATE_IDLE; 789 break; 790 791 case SIW_QP_STATE_CLOSING: 792 /* 793 * The LLP may already moved the QP to closing 794 * due to graceful peer close init 795 */ 796 break; 797 798 case SIW_QP_STATE_ERROR: 799 /* 800 * QP was moved to CLOSING by LLP event 801 * not yet seen by user. 802 */ 803 qp->attrs.state = SIW_QP_STATE_ERROR; 804 805 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) 806 siw_sq_flush(qp); 807 808 siw_rq_flush(qp); 809 break; 810 811 default: 812 siw_dbg_qp(qp, "state transition undefined: %s => %s\n", 813 siw_qp_state_to_string[qp->attrs.state], 814 siw_qp_state_to_string[attrs->state]); 815 816 rv = -ECONNABORTED; 817 } 818 return rv; 819 } 820 821 /* 822 * Caller must hold qp->state_lock 823 */ 824 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs, 825 enum siw_qp_attr_mask mask) 826 { 827 int drop_conn = 0, rv = 0; 828 829 if (!mask) 830 return 0; 831 832 siw_dbg_qp(qp, "state: %s => %s\n", 833 siw_qp_state_to_string[qp->attrs.state], 834 siw_qp_state_to_string[attrs->state]); 835 836 if (mask != SIW_QP_ATTR_STATE) 837 siw_qp_modify_nonstate(qp, attrs, mask); 838 839 if (!(mask & SIW_QP_ATTR_STATE)) 840 return 0; 841 842 switch (qp->attrs.state) { 843 case SIW_QP_STATE_IDLE: 844 case SIW_QP_STATE_RTR: 845 rv = siw_qp_nextstate_from_idle(qp, attrs, mask); 846 break; 847 848 case SIW_QP_STATE_RTS: 849 drop_conn = siw_qp_nextstate_from_rts(qp, attrs); 850 break; 851 852 case SIW_QP_STATE_TERMINATE: 853 siw_qp_nextstate_from_term(qp, attrs); 854 break; 855 856 case SIW_QP_STATE_CLOSING: 857 siw_qp_nextstate_from_close(qp, attrs); 858 break; 859 default: 860 break; 861 } 862 if (drop_conn) 863 siw_qp_cm_drop(qp, 0); 864 865 return rv; 866 } 867 868 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe) 869 { 870 rreq->id = sqe->id; 871 rreq->opcode = sqe->opcode; 872 rreq->sge[0].laddr = sqe->sge[0].laddr; 873 rreq->sge[0].length = sqe->sge[0].length; 874 rreq->sge[0].lkey = sqe->sge[0].lkey; 875 rreq->sge[1].lkey = sqe->sge[1].lkey; 876 rreq->flags = sqe->flags | SIW_WQE_VALID; 877 rreq->num_sge = 1; 878 } 879 880 /* 881 * Must be called with SQ locked. 882 * To avoid complete SQ starvation by constant inbound READ requests, 883 * the active IRQ will not be served after qp->irq_burst, if the 884 * SQ has pending work. 885 */ 886 int siw_activate_tx(struct siw_qp *qp) 887 { 888 struct siw_sqe *irqe, *sqe; 889 struct siw_wqe *wqe = tx_wqe(qp); 890 int rv = 1; 891 892 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; 893 894 if (irqe->flags & SIW_WQE_VALID) { 895 sqe = sq_get_next(qp); 896 897 /* 898 * Avoid local WQE processing starvation in case 899 * of constant inbound READ request stream 900 */ 901 if (sqe && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { 902 qp->irq_burst = 0; 903 goto skip_irq; 904 } 905 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 906 wqe->wr_status = SIW_WR_QUEUED; 907 908 /* start READ RESPONSE */ 909 wqe->sqe.opcode = SIW_OP_READ_RESPONSE; 910 wqe->sqe.flags = 0; 911 if (irqe->num_sge) { 912 wqe->sqe.num_sge = 1; 913 wqe->sqe.sge[0].length = irqe->sge[0].length; 914 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr; 915 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey; 916 } else { 917 wqe->sqe.num_sge = 0; 918 } 919 920 /* Retain original RREQ's message sequence number for 921 * potential error reporting cases. 922 */ 923 wqe->sqe.sge[1].length = irqe->sge[1].length; 924 925 wqe->sqe.rkey = irqe->rkey; 926 wqe->sqe.raddr = irqe->raddr; 927 928 wqe->processed = 0; 929 qp->irq_get++; 930 931 /* mark current IRQ entry free */ 932 smp_store_mb(irqe->flags, 0); 933 934 goto out; 935 } 936 sqe = sq_get_next(qp); 937 if (sqe) { 938 skip_irq: 939 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); 940 wqe->wr_status = SIW_WR_QUEUED; 941 942 /* First copy SQE to kernel private memory */ 943 memcpy(&wqe->sqe, sqe, sizeof(*sqe)); 944 945 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) { 946 rv = -EINVAL; 947 goto out; 948 } 949 if (wqe->sqe.flags & SIW_WQE_INLINE) { 950 if (wqe->sqe.opcode != SIW_OP_SEND && 951 wqe->sqe.opcode != SIW_OP_WRITE) { 952 rv = -EINVAL; 953 goto out; 954 } 955 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) { 956 rv = -EINVAL; 957 goto out; 958 } 959 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1]; 960 wqe->sqe.sge[0].lkey = 0; 961 wqe->sqe.num_sge = 1; 962 } 963 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) { 964 /* A READ cannot be fenced */ 965 if (unlikely(wqe->sqe.opcode == SIW_OP_READ || 966 wqe->sqe.opcode == 967 SIW_OP_READ_LOCAL_INV)) { 968 siw_dbg_qp(qp, "cannot fence read\n"); 969 rv = -EINVAL; 970 goto out; 971 } 972 spin_lock(&qp->orq_lock); 973 974 if (!siw_orq_empty(qp)) { 975 qp->tx_ctx.orq_fence = 1; 976 rv = 0; 977 } 978 spin_unlock(&qp->orq_lock); 979 980 } else if (wqe->sqe.opcode == SIW_OP_READ || 981 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) { 982 struct siw_sqe *rreq; 983 984 wqe->sqe.num_sge = 1; 985 986 spin_lock(&qp->orq_lock); 987 988 rreq = orq_get_free(qp); 989 if (rreq) { 990 /* 991 * Make an immediate copy in ORQ to be ready 992 * to process loopback READ reply 993 */ 994 siw_read_to_orq(rreq, &wqe->sqe); 995 qp->orq_put++; 996 } else { 997 qp->tx_ctx.orq_fence = 1; 998 rv = 0; 999 } 1000 spin_unlock(&qp->orq_lock); 1001 } 1002 1003 /* Clear SQE, can be re-used by application */ 1004 smp_store_mb(sqe->flags, 0); 1005 qp->sq_get++; 1006 } else { 1007 rv = 0; 1008 } 1009 out: 1010 if (unlikely(rv < 0)) { 1011 siw_dbg_qp(qp, "error %d\n", rv); 1012 wqe->wr_status = SIW_WR_IDLE; 1013 } 1014 return rv; 1015 } 1016 1017 /* 1018 * Check if current CQ state qualifies for calling CQ completion 1019 * handler. Must be called with CQ lock held. 1020 */ 1021 static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags) 1022 { 1023 u32 cq_notify; 1024 1025 if (!cq->base_cq.comp_handler) 1026 return false; 1027 1028 /* Read application shared notification state */ 1029 cq_notify = READ_ONCE(cq->notify->flags); 1030 1031 if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) || 1032 ((cq_notify & SIW_NOTIFY_SOLICITED) && 1033 (flags & SIW_WQE_SOLICITED))) { 1034 /* 1035 * CQ notification is one-shot: Since the 1036 * current CQE causes user notification, 1037 * the CQ gets dis-aremd and must be re-aremd 1038 * by the user for a new notification. 1039 */ 1040 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT); 1041 1042 return true; 1043 } 1044 return false; 1045 } 1046 1047 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, 1048 enum siw_wc_status status) 1049 { 1050 struct siw_cq *cq = qp->scq; 1051 int rv = 0; 1052 1053 if (cq) { 1054 u32 sqe_flags = sqe->flags; 1055 struct siw_cqe *cqe; 1056 u32 idx; 1057 unsigned long flags; 1058 1059 spin_lock_irqsave(&cq->lock, flags); 1060 1061 idx = cq->cq_put % cq->num_cqe; 1062 cqe = &cq->queue[idx]; 1063 1064 if (!READ_ONCE(cqe->flags)) { 1065 bool notify; 1066 1067 cqe->id = sqe->id; 1068 cqe->opcode = sqe->opcode; 1069 cqe->status = status; 1070 cqe->imm_data = 0; 1071 cqe->bytes = bytes; 1072 1073 if (rdma_is_kernel_res(&cq->base_cq.res)) 1074 cqe->base_qp = &qp->base_qp; 1075 else 1076 cqe->qp_id = qp_id(qp); 1077 1078 /* mark CQE valid for application */ 1079 WRITE_ONCE(cqe->flags, SIW_WQE_VALID); 1080 /* recycle SQE */ 1081 smp_store_mb(sqe->flags, 0); 1082 1083 cq->cq_put++; 1084 notify = siw_cq_notify_now(cq, sqe_flags); 1085 1086 spin_unlock_irqrestore(&cq->lock, flags); 1087 1088 if (notify) { 1089 siw_dbg_cq(cq, "Call completion handler\n"); 1090 cq->base_cq.comp_handler(&cq->base_cq, 1091 cq->base_cq.cq_context); 1092 } 1093 } else { 1094 spin_unlock_irqrestore(&cq->lock, flags); 1095 rv = -ENOMEM; 1096 siw_cq_event(cq, IB_EVENT_CQ_ERR); 1097 } 1098 } else { 1099 /* recycle SQE */ 1100 smp_store_mb(sqe->flags, 0); 1101 } 1102 return rv; 1103 } 1104 1105 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, 1106 u32 inval_stag, enum siw_wc_status status) 1107 { 1108 struct siw_cq *cq = qp->rcq; 1109 int rv = 0; 1110 1111 if (cq) { 1112 struct siw_cqe *cqe; 1113 u32 idx; 1114 unsigned long flags; 1115 1116 spin_lock_irqsave(&cq->lock, flags); 1117 1118 idx = cq->cq_put % cq->num_cqe; 1119 cqe = &cq->queue[idx]; 1120 1121 if (!READ_ONCE(cqe->flags)) { 1122 bool notify; 1123 u8 cqe_flags = SIW_WQE_VALID; 1124 1125 cqe->id = rqe->id; 1126 cqe->opcode = SIW_OP_RECEIVE; 1127 cqe->status = status; 1128 cqe->imm_data = 0; 1129 cqe->bytes = bytes; 1130 1131 if (rdma_is_kernel_res(&cq->base_cq.res)) { 1132 cqe->base_qp = &qp->base_qp; 1133 if (inval_stag) { 1134 cqe_flags |= SIW_WQE_REM_INVAL; 1135 cqe->inval_stag = inval_stag; 1136 } 1137 } else { 1138 cqe->qp_id = qp_id(qp); 1139 } 1140 /* mark CQE valid for application */ 1141 WRITE_ONCE(cqe->flags, cqe_flags); 1142 /* recycle RQE */ 1143 smp_store_mb(rqe->flags, 0); 1144 1145 cq->cq_put++; 1146 notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED); 1147 1148 spin_unlock_irqrestore(&cq->lock, flags); 1149 1150 if (notify) { 1151 siw_dbg_cq(cq, "Call completion handler\n"); 1152 cq->base_cq.comp_handler(&cq->base_cq, 1153 cq->base_cq.cq_context); 1154 } 1155 } else { 1156 spin_unlock_irqrestore(&cq->lock, flags); 1157 rv = -ENOMEM; 1158 siw_cq_event(cq, IB_EVENT_CQ_ERR); 1159 } 1160 } else { 1161 /* recycle RQE */ 1162 smp_store_mb(rqe->flags, 0); 1163 } 1164 return rv; 1165 } 1166 1167 /* 1168 * siw_sq_flush() 1169 * 1170 * Flush SQ and ORRQ entries to CQ. 1171 * 1172 * Must be called with QP state write lock held. 1173 * Therefore, SQ and ORQ lock must not be taken. 1174 */ 1175 void siw_sq_flush(struct siw_qp *qp) 1176 { 1177 struct siw_sqe *sqe; 1178 struct siw_wqe *wqe = tx_wqe(qp); 1179 int async_event = 0; 1180 1181 /* 1182 * Start with completing any work currently on the ORQ 1183 */ 1184 while (qp->attrs.orq_size) { 1185 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; 1186 if (!READ_ONCE(sqe->flags)) 1187 break; 1188 1189 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1190 break; 1191 1192 WRITE_ONCE(sqe->flags, 0); 1193 qp->orq_get++; 1194 } 1195 /* 1196 * Flush an in-progress WQE if present 1197 */ 1198 if (wqe->wr_status != SIW_WR_IDLE) { 1199 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n", 1200 tx_type(wqe), wqe->wr_status); 1201 1202 siw_wqe_put_mem(wqe, tx_type(wqe)); 1203 1204 if (tx_type(wqe) != SIW_OP_READ_RESPONSE && 1205 ((tx_type(wqe) != SIW_OP_READ && 1206 tx_type(wqe) != SIW_OP_READ_LOCAL_INV) || 1207 wqe->wr_status == SIW_WR_QUEUED)) 1208 /* 1209 * An in-progress Read Request is already in 1210 * the ORQ 1211 */ 1212 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, 1213 SIW_WC_WR_FLUSH_ERR); 1214 1215 wqe->wr_status = SIW_WR_IDLE; 1216 } 1217 /* 1218 * Flush the Send Queue 1219 */ 1220 while (qp->attrs.sq_size) { 1221 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; 1222 if (!READ_ONCE(sqe->flags)) 1223 break; 1224 1225 async_event = 1; 1226 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1227 /* 1228 * Shall IB_EVENT_SQ_DRAINED be supressed if work 1229 * completion fails? 1230 */ 1231 break; 1232 1233 WRITE_ONCE(sqe->flags, 0); 1234 qp->sq_get++; 1235 } 1236 if (async_event) 1237 siw_qp_event(qp, IB_EVENT_SQ_DRAINED); 1238 } 1239 1240 /* 1241 * siw_rq_flush() 1242 * 1243 * Flush recv queue entries to CQ. Also 1244 * takes care of pending active tagged and untagged 1245 * inbound transfers, which have target memory 1246 * referenced. 1247 * 1248 * Must be called with QP state write lock held. 1249 * Therefore, RQ lock must not be taken. 1250 */ 1251 void siw_rq_flush(struct siw_qp *qp) 1252 { 1253 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; 1254 1255 /* 1256 * Flush an in-progress untagged operation if present 1257 */ 1258 if (wqe->wr_status != SIW_WR_IDLE) { 1259 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n", 1260 rx_type(wqe), wqe->wr_status); 1261 1262 siw_wqe_put_mem(wqe, rx_type(wqe)); 1263 1264 if (rx_type(wqe) == SIW_OP_RECEIVE) { 1265 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, 1266 0, SIW_WC_WR_FLUSH_ERR); 1267 } else if (rx_type(wqe) != SIW_OP_READ && 1268 rx_type(wqe) != SIW_OP_READ_RESPONSE && 1269 rx_type(wqe) != SIW_OP_WRITE) { 1270 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); 1271 } 1272 wqe->wr_status = SIW_WR_IDLE; 1273 } 1274 wqe = &qp->rx_tagged.wqe_active; 1275 1276 if (wqe->wr_status != SIW_WR_IDLE) { 1277 siw_wqe_put_mem(wqe, rx_type(wqe)); 1278 wqe->wr_status = SIW_WR_IDLE; 1279 } 1280 /* 1281 * Flush the Receive Queue 1282 */ 1283 while (qp->attrs.rq_size) { 1284 struct siw_rqe *rqe = 1285 &qp->recvq[qp->rq_get % qp->attrs.rq_size]; 1286 1287 if (!READ_ONCE(rqe->flags)) 1288 break; 1289 1290 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0) 1291 break; 1292 1293 WRITE_ONCE(rqe->flags, 0); 1294 qp->rq_get++; 1295 } 1296 } 1297 1298 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) 1299 { 1300 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, 1301 GFP_KERNEL); 1302 1303 if (!rv) { 1304 kref_init(&qp->ref); 1305 qp->sdev = sdev; 1306 siw_dbg_qp(qp, "new QP\n"); 1307 } 1308 return rv; 1309 } 1310 1311 void siw_free_qp(struct kref *ref) 1312 { 1313 struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref); 1314 struct siw_device *sdev = qp->sdev; 1315 unsigned long flags; 1316 1317 if (qp->cep) 1318 siw_cep_put(qp->cep); 1319 1320 found = xa_erase(&sdev->qp_xa, qp_id(qp)); 1321 WARN_ON(found != qp); 1322 spin_lock_irqsave(&sdev->lock, flags); 1323 list_del(&qp->devq); 1324 spin_unlock_irqrestore(&sdev->lock, flags); 1325 1326 vfree(qp->sendq); 1327 vfree(qp->recvq); 1328 vfree(qp->irq); 1329 vfree(qp->orq); 1330 1331 siw_put_tx_cpu(qp->tx_cpu); 1332 1333 atomic_dec(&sdev->num_qp); 1334 siw_dbg_qp(qp, "free QP\n"); 1335 kfree_rcu(qp, rcu); 1336 } 1337