1 /* 2 * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. 3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/io.h> 35 36 #include "qib.h" 37 38 /* cut down ridiculously long IB macro names */ 39 #define OP(x) IB_OPCODE_RC_##x 40 41 static void rc_timeout(unsigned long arg); 42 43 static u32 restart_sge(struct qib_sge_state *ss, struct qib_swqe *wqe, 44 u32 psn, u32 pmtu) 45 { 46 u32 len; 47 48 len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu; 49 ss->sge = wqe->sg_list[0]; 50 ss->sg_list = wqe->sg_list + 1; 51 ss->num_sge = wqe->wr.num_sge; 52 ss->total_len = wqe->length; 53 qib_skip_sge(ss, len, 0); 54 return wqe->length - len; 55 } 56 57 static void start_timer(struct qib_qp *qp) 58 { 59 qp->s_flags |= QIB_S_TIMER; 60 qp->s_timer.function = rc_timeout; 61 /* 4.096 usec. * (1 << qp->timeout) */ 62 qp->s_timer.expires = jiffies + qp->timeout_jiffies; 63 add_timer(&qp->s_timer); 64 } 65 66 /** 67 * qib_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) 68 * @dev: the device for this QP 69 * @qp: a pointer to the QP 70 * @ohdr: a pointer to the IB header being constructed 71 * @pmtu: the path MTU 72 * 73 * Return 1 if constructed; otherwise, return 0. 74 * Note that we are in the responder's side of the QP context. 75 * Note the QP s_lock must be held. 76 */ 77 static int qib_make_rc_ack(struct qib_ibdev *dev, struct qib_qp *qp, 78 struct qib_other_headers *ohdr, u32 pmtu) 79 { 80 struct qib_ack_entry *e; 81 u32 hwords; 82 u32 len; 83 u32 bth0; 84 u32 bth2; 85 86 /* Don't send an ACK if we aren't supposed to. */ 87 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 88 goto bail; 89 90 /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 91 hwords = 5; 92 93 switch (qp->s_ack_state) { 94 case OP(RDMA_READ_RESPONSE_LAST): 95 case OP(RDMA_READ_RESPONSE_ONLY): 96 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 97 if (e->rdma_sge.mr) { 98 qib_put_mr(e->rdma_sge.mr); 99 e->rdma_sge.mr = NULL; 100 } 101 /* FALLTHROUGH */ 102 case OP(ATOMIC_ACKNOWLEDGE): 103 /* 104 * We can increment the tail pointer now that the last 105 * response has been sent instead of only being 106 * constructed. 107 */ 108 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) 109 qp->s_tail_ack_queue = 0; 110 /* FALLTHROUGH */ 111 case OP(SEND_ONLY): 112 case OP(ACKNOWLEDGE): 113 /* Check for no next entry in the queue. */ 114 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { 115 if (qp->s_flags & QIB_S_ACK_PENDING) 116 goto normal; 117 goto bail; 118 } 119 120 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 121 if (e->opcode == OP(RDMA_READ_REQUEST)) { 122 /* 123 * If a RDMA read response is being resent and 124 * we haven't seen the duplicate request yet, 125 * then stop sending the remaining responses the 126 * responder has seen until the requester resends it. 127 */ 128 len = e->rdma_sge.sge_length; 129 if (len && !e->rdma_sge.mr) { 130 qp->s_tail_ack_queue = qp->r_head_ack_queue; 131 goto bail; 132 } 133 /* Copy SGE state in case we need to resend */ 134 qp->s_rdma_mr = e->rdma_sge.mr; 135 if (qp->s_rdma_mr) 136 qib_get_mr(qp->s_rdma_mr); 137 qp->s_ack_rdma_sge.sge = e->rdma_sge; 138 qp->s_ack_rdma_sge.num_sge = 1; 139 qp->s_cur_sge = &qp->s_ack_rdma_sge; 140 if (len > pmtu) { 141 len = pmtu; 142 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); 143 } else { 144 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); 145 e->sent = 1; 146 } 147 ohdr->u.aeth = qib_compute_aeth(qp); 148 hwords++; 149 qp->s_ack_rdma_psn = e->psn; 150 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; 151 } else { 152 /* COMPARE_SWAP or FETCH_ADD */ 153 qp->s_cur_sge = NULL; 154 len = 0; 155 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 156 ohdr->u.at.aeth = qib_compute_aeth(qp); 157 ohdr->u.at.atomic_ack_eth[0] = 158 cpu_to_be32(e->atomic_data >> 32); 159 ohdr->u.at.atomic_ack_eth[1] = 160 cpu_to_be32(e->atomic_data); 161 hwords += sizeof(ohdr->u.at) / sizeof(u32); 162 bth2 = e->psn & QIB_PSN_MASK; 163 e->sent = 1; 164 } 165 bth0 = qp->s_ack_state << 24; 166 break; 167 168 case OP(RDMA_READ_RESPONSE_FIRST): 169 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); 170 /* FALLTHROUGH */ 171 case OP(RDMA_READ_RESPONSE_MIDDLE): 172 qp->s_cur_sge = &qp->s_ack_rdma_sge; 173 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; 174 if (qp->s_rdma_mr) 175 qib_get_mr(qp->s_rdma_mr); 176 len = qp->s_ack_rdma_sge.sge.sge_length; 177 if (len > pmtu) 178 len = pmtu; 179 else { 180 ohdr->u.aeth = qib_compute_aeth(qp); 181 hwords++; 182 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 183 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 184 e->sent = 1; 185 } 186 bth0 = qp->s_ack_state << 24; 187 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; 188 break; 189 190 default: 191 normal: 192 /* 193 * Send a regular ACK. 194 * Set the s_ack_state so we wait until after sending 195 * the ACK before setting s_ack_state to ACKNOWLEDGE 196 * (see above). 197 */ 198 qp->s_ack_state = OP(SEND_ONLY); 199 qp->s_flags &= ~QIB_S_ACK_PENDING; 200 qp->s_cur_sge = NULL; 201 if (qp->s_nak_state) 202 ohdr->u.aeth = 203 cpu_to_be32((qp->r_msn & QIB_MSN_MASK) | 204 (qp->s_nak_state << 205 QIB_AETH_CREDIT_SHIFT)); 206 else 207 ohdr->u.aeth = qib_compute_aeth(qp); 208 hwords++; 209 len = 0; 210 bth0 = OP(ACKNOWLEDGE) << 24; 211 bth2 = qp->s_ack_psn & QIB_PSN_MASK; 212 } 213 qp->s_rdma_ack_cnt++; 214 qp->s_hdrwords = hwords; 215 qp->s_cur_size = len; 216 qib_make_ruc_header(qp, ohdr, bth0, bth2); 217 return 1; 218 219 bail: 220 qp->s_ack_state = OP(ACKNOWLEDGE); 221 qp->s_flags &= ~(QIB_S_RESP_PENDING | QIB_S_ACK_PENDING); 222 return 0; 223 } 224 225 /** 226 * qib_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) 227 * @qp: a pointer to the QP 228 * 229 * Return 1 if constructed; otherwise, return 0. 230 */ 231 int qib_make_rc_req(struct qib_qp *qp) 232 { 233 struct qib_ibdev *dev = to_idev(qp->ibqp.device); 234 struct qib_other_headers *ohdr; 235 struct qib_sge_state *ss; 236 struct qib_swqe *wqe; 237 u32 hwords; 238 u32 len; 239 u32 bth0; 240 u32 bth2; 241 u32 pmtu = qp->pmtu; 242 char newreq; 243 unsigned long flags; 244 int ret = 0; 245 int delta; 246 247 ohdr = &qp->s_hdr->u.oth; 248 if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) 249 ohdr = &qp->s_hdr->u.l.oth; 250 251 /* 252 * The lock is needed to synchronize between the sending tasklet, 253 * the receive interrupt handler, and timeout resends. 254 */ 255 spin_lock_irqsave(&qp->s_lock, flags); 256 257 /* Sending responses has higher priority over sending requests. */ 258 if ((qp->s_flags & QIB_S_RESP_PENDING) && 259 qib_make_rc_ack(dev, qp, ohdr, pmtu)) 260 goto done; 261 262 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_SEND_OK)) { 263 if (!(ib_qib_state_ops[qp->state] & QIB_FLUSH_SEND)) 264 goto bail; 265 /* We are in the error state, flush the work request. */ 266 if (qp->s_last == qp->s_head) 267 goto bail; 268 /* If DMAs are in progress, we can't flush immediately. */ 269 if (atomic_read(&qp->s_dma_busy)) { 270 qp->s_flags |= QIB_S_WAIT_DMA; 271 goto bail; 272 } 273 wqe = get_swqe_ptr(qp, qp->s_last); 274 qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ? 275 IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR); 276 /* will get called again */ 277 goto done; 278 } 279 280 if (qp->s_flags & (QIB_S_WAIT_RNR | QIB_S_WAIT_ACK)) 281 goto bail; 282 283 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { 284 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { 285 qp->s_flags |= QIB_S_WAIT_PSN; 286 goto bail; 287 } 288 qp->s_sending_psn = qp->s_psn; 289 qp->s_sending_hpsn = qp->s_psn - 1; 290 } 291 292 /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 293 hwords = 5; 294 bth0 = 0; 295 296 /* Send a request. */ 297 wqe = get_swqe_ptr(qp, qp->s_cur); 298 switch (qp->s_state) { 299 default: 300 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_NEXT_SEND_OK)) 301 goto bail; 302 /* 303 * Resend an old request or start a new one. 304 * 305 * We keep track of the current SWQE so that 306 * we don't reset the "furthest progress" state 307 * if we need to back up. 308 */ 309 newreq = 0; 310 if (qp->s_cur == qp->s_tail) { 311 /* Check if send work queue is empty. */ 312 if (qp->s_tail == qp->s_head) 313 goto bail; 314 /* 315 * If a fence is requested, wait for previous 316 * RDMA read and atomic operations to finish. 317 */ 318 if ((wqe->wr.send_flags & IB_SEND_FENCE) && 319 qp->s_num_rd_atomic) { 320 qp->s_flags |= QIB_S_WAIT_FENCE; 321 goto bail; 322 } 323 wqe->psn = qp->s_next_psn; 324 newreq = 1; 325 } 326 /* 327 * Note that we have to be careful not to modify the 328 * original work request since we may need to resend 329 * it. 330 */ 331 len = wqe->length; 332 ss = &qp->s_sge; 333 bth2 = qp->s_psn & QIB_PSN_MASK; 334 switch (wqe->wr.opcode) { 335 case IB_WR_SEND: 336 case IB_WR_SEND_WITH_IMM: 337 /* If no credit, return. */ 338 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) && 339 qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { 340 qp->s_flags |= QIB_S_WAIT_SSN_CREDIT; 341 goto bail; 342 } 343 wqe->lpsn = wqe->psn; 344 if (len > pmtu) { 345 wqe->lpsn += (len - 1) / pmtu; 346 qp->s_state = OP(SEND_FIRST); 347 len = pmtu; 348 break; 349 } 350 if (wqe->wr.opcode == IB_WR_SEND) 351 qp->s_state = OP(SEND_ONLY); 352 else { 353 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); 354 /* Immediate data comes after the BTH */ 355 ohdr->u.imm_data = wqe->wr.ex.imm_data; 356 hwords += 1; 357 } 358 if (wqe->wr.send_flags & IB_SEND_SOLICITED) 359 bth0 |= IB_BTH_SOLICITED; 360 bth2 |= IB_BTH_REQ_ACK; 361 if (++qp->s_cur == qp->s_size) 362 qp->s_cur = 0; 363 break; 364 365 case IB_WR_RDMA_WRITE: 366 if (newreq && !(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 367 qp->s_lsn++; 368 /* FALLTHROUGH */ 369 case IB_WR_RDMA_WRITE_WITH_IMM: 370 /* If no credit, return. */ 371 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) && 372 qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { 373 qp->s_flags |= QIB_S_WAIT_SSN_CREDIT; 374 goto bail; 375 } 376 377 ohdr->u.rc.reth.vaddr = 378 cpu_to_be64(wqe->rdma_wr.remote_addr); 379 ohdr->u.rc.reth.rkey = 380 cpu_to_be32(wqe->rdma_wr.rkey); 381 ohdr->u.rc.reth.length = cpu_to_be32(len); 382 hwords += sizeof(struct ib_reth) / sizeof(u32); 383 wqe->lpsn = wqe->psn; 384 if (len > pmtu) { 385 wqe->lpsn += (len - 1) / pmtu; 386 qp->s_state = OP(RDMA_WRITE_FIRST); 387 len = pmtu; 388 break; 389 } 390 if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE) 391 qp->s_state = OP(RDMA_WRITE_ONLY); 392 else { 393 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 394 /* Immediate data comes after RETH */ 395 ohdr->u.rc.imm_data = 396 wqe->rdma_wr.wr.ex.imm_data; 397 hwords += 1; 398 if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED) 399 bth0 |= IB_BTH_SOLICITED; 400 } 401 bth2 |= IB_BTH_REQ_ACK; 402 if (++qp->s_cur == qp->s_size) 403 qp->s_cur = 0; 404 break; 405 406 case IB_WR_RDMA_READ: 407 /* 408 * Don't allow more operations to be started 409 * than the QP limits allow. 410 */ 411 if (newreq) { 412 if (qp->s_num_rd_atomic >= 413 qp->s_max_rd_atomic) { 414 qp->s_flags |= QIB_S_WAIT_RDMAR; 415 goto bail; 416 } 417 qp->s_num_rd_atomic++; 418 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 419 qp->s_lsn++; 420 /* 421 * Adjust s_next_psn to count the 422 * expected number of responses. 423 */ 424 if (len > pmtu) 425 qp->s_next_psn += (len - 1) / pmtu; 426 wqe->lpsn = qp->s_next_psn++; 427 } 428 429 ohdr->u.rc.reth.vaddr = 430 cpu_to_be64(wqe->rdma_wr.remote_addr); 431 ohdr->u.rc.reth.rkey = 432 cpu_to_be32(wqe->rdma_wr.rkey); 433 ohdr->u.rc.reth.length = cpu_to_be32(len); 434 qp->s_state = OP(RDMA_READ_REQUEST); 435 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 436 ss = NULL; 437 len = 0; 438 bth2 |= IB_BTH_REQ_ACK; 439 if (++qp->s_cur == qp->s_size) 440 qp->s_cur = 0; 441 break; 442 443 case IB_WR_ATOMIC_CMP_AND_SWP: 444 case IB_WR_ATOMIC_FETCH_AND_ADD: 445 /* 446 * Don't allow more operations to be started 447 * than the QP limits allow. 448 */ 449 if (newreq) { 450 if (qp->s_num_rd_atomic >= 451 qp->s_max_rd_atomic) { 452 qp->s_flags |= QIB_S_WAIT_RDMAR; 453 goto bail; 454 } 455 qp->s_num_rd_atomic++; 456 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) 457 qp->s_lsn++; 458 wqe->lpsn = wqe->psn; 459 } 460 if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 461 qp->s_state = OP(COMPARE_SWAP); 462 ohdr->u.atomic_eth.swap_data = cpu_to_be64( 463 wqe->atomic_wr.swap); 464 ohdr->u.atomic_eth.compare_data = cpu_to_be64( 465 wqe->atomic_wr.compare_add); 466 } else { 467 qp->s_state = OP(FETCH_ADD); 468 ohdr->u.atomic_eth.swap_data = cpu_to_be64( 469 wqe->atomic_wr.compare_add); 470 ohdr->u.atomic_eth.compare_data = 0; 471 } 472 ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32( 473 wqe->atomic_wr.remote_addr >> 32); 474 ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32( 475 wqe->atomic_wr.remote_addr); 476 ohdr->u.atomic_eth.rkey = cpu_to_be32( 477 wqe->atomic_wr.rkey); 478 hwords += sizeof(struct ib_atomic_eth) / sizeof(u32); 479 ss = NULL; 480 len = 0; 481 bth2 |= IB_BTH_REQ_ACK; 482 if (++qp->s_cur == qp->s_size) 483 qp->s_cur = 0; 484 break; 485 486 default: 487 goto bail; 488 } 489 qp->s_sge.sge = wqe->sg_list[0]; 490 qp->s_sge.sg_list = wqe->sg_list + 1; 491 qp->s_sge.num_sge = wqe->wr.num_sge; 492 qp->s_sge.total_len = wqe->length; 493 qp->s_len = wqe->length; 494 if (newreq) { 495 qp->s_tail++; 496 if (qp->s_tail >= qp->s_size) 497 qp->s_tail = 0; 498 } 499 if (wqe->wr.opcode == IB_WR_RDMA_READ) 500 qp->s_psn = wqe->lpsn + 1; 501 else { 502 qp->s_psn++; 503 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 504 qp->s_next_psn = qp->s_psn; 505 } 506 break; 507 508 case OP(RDMA_READ_RESPONSE_FIRST): 509 /* 510 * qp->s_state is normally set to the opcode of the 511 * last packet constructed for new requests and therefore 512 * is never set to RDMA read response. 513 * RDMA_READ_RESPONSE_FIRST is used by the ACK processing 514 * thread to indicate a SEND needs to be restarted from an 515 * earlier PSN without interferring with the sending thread. 516 * See qib_restart_rc(). 517 */ 518 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 519 /* FALLTHROUGH */ 520 case OP(SEND_FIRST): 521 qp->s_state = OP(SEND_MIDDLE); 522 /* FALLTHROUGH */ 523 case OP(SEND_MIDDLE): 524 bth2 = qp->s_psn++ & QIB_PSN_MASK; 525 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 526 qp->s_next_psn = qp->s_psn; 527 ss = &qp->s_sge; 528 len = qp->s_len; 529 if (len > pmtu) { 530 len = pmtu; 531 break; 532 } 533 if (wqe->wr.opcode == IB_WR_SEND) 534 qp->s_state = OP(SEND_LAST); 535 else { 536 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 537 /* Immediate data comes after the BTH */ 538 ohdr->u.imm_data = wqe->wr.ex.imm_data; 539 hwords += 1; 540 } 541 if (wqe->wr.send_flags & IB_SEND_SOLICITED) 542 bth0 |= IB_BTH_SOLICITED; 543 bth2 |= IB_BTH_REQ_ACK; 544 qp->s_cur++; 545 if (qp->s_cur >= qp->s_size) 546 qp->s_cur = 0; 547 break; 548 549 case OP(RDMA_READ_RESPONSE_LAST): 550 /* 551 * qp->s_state is normally set to the opcode of the 552 * last packet constructed for new requests and therefore 553 * is never set to RDMA read response. 554 * RDMA_READ_RESPONSE_LAST is used by the ACK processing 555 * thread to indicate a RDMA write needs to be restarted from 556 * an earlier PSN without interferring with the sending thread. 557 * See qib_restart_rc(). 558 */ 559 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 560 /* FALLTHROUGH */ 561 case OP(RDMA_WRITE_FIRST): 562 qp->s_state = OP(RDMA_WRITE_MIDDLE); 563 /* FALLTHROUGH */ 564 case OP(RDMA_WRITE_MIDDLE): 565 bth2 = qp->s_psn++ & QIB_PSN_MASK; 566 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) 567 qp->s_next_psn = qp->s_psn; 568 ss = &qp->s_sge; 569 len = qp->s_len; 570 if (len > pmtu) { 571 len = pmtu; 572 break; 573 } 574 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 575 qp->s_state = OP(RDMA_WRITE_LAST); 576 else { 577 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 578 /* Immediate data comes after the BTH */ 579 ohdr->u.imm_data = wqe->wr.ex.imm_data; 580 hwords += 1; 581 if (wqe->wr.send_flags & IB_SEND_SOLICITED) 582 bth0 |= IB_BTH_SOLICITED; 583 } 584 bth2 |= IB_BTH_REQ_ACK; 585 qp->s_cur++; 586 if (qp->s_cur >= qp->s_size) 587 qp->s_cur = 0; 588 break; 589 590 case OP(RDMA_READ_RESPONSE_MIDDLE): 591 /* 592 * qp->s_state is normally set to the opcode of the 593 * last packet constructed for new requests and therefore 594 * is never set to RDMA read response. 595 * RDMA_READ_RESPONSE_MIDDLE is used by the ACK processing 596 * thread to indicate a RDMA read needs to be restarted from 597 * an earlier PSN without interferring with the sending thread. 598 * See qib_restart_rc(). 599 */ 600 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; 601 ohdr->u.rc.reth.vaddr = 602 cpu_to_be64(wqe->rdma_wr.remote_addr + len); 603 ohdr->u.rc.reth.rkey = 604 cpu_to_be32(wqe->rdma_wr.rkey); 605 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); 606 qp->s_state = OP(RDMA_READ_REQUEST); 607 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 608 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; 609 qp->s_psn = wqe->lpsn + 1; 610 ss = NULL; 611 len = 0; 612 qp->s_cur++; 613 if (qp->s_cur == qp->s_size) 614 qp->s_cur = 0; 615 break; 616 } 617 qp->s_sending_hpsn = bth2; 618 delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8; 619 if (delta && delta % QIB_PSN_CREDIT == 0) 620 bth2 |= IB_BTH_REQ_ACK; 621 if (qp->s_flags & QIB_S_SEND_ONE) { 622 qp->s_flags &= ~QIB_S_SEND_ONE; 623 qp->s_flags |= QIB_S_WAIT_ACK; 624 bth2 |= IB_BTH_REQ_ACK; 625 } 626 qp->s_len -= len; 627 qp->s_hdrwords = hwords; 628 qp->s_cur_sge = ss; 629 qp->s_cur_size = len; 630 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); 631 done: 632 ret = 1; 633 goto unlock; 634 635 bail: 636 qp->s_flags &= ~QIB_S_BUSY; 637 unlock: 638 spin_unlock_irqrestore(&qp->s_lock, flags); 639 return ret; 640 } 641 642 /** 643 * qib_send_rc_ack - Construct an ACK packet and send it 644 * @qp: a pointer to the QP 645 * 646 * This is called from qib_rc_rcv() and qib_kreceive(). 647 * Note that RDMA reads and atomics are handled in the 648 * send side QP state and tasklet. 649 */ 650 void qib_send_rc_ack(struct qib_qp *qp) 651 { 652 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 653 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 654 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 655 u64 pbc; 656 u16 lrh0; 657 u32 bth0; 658 u32 hwords; 659 u32 pbufn; 660 u32 __iomem *piobuf; 661 struct qib_ib_header hdr; 662 struct qib_other_headers *ohdr; 663 u32 control; 664 unsigned long flags; 665 666 spin_lock_irqsave(&qp->s_lock, flags); 667 668 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 669 goto unlock; 670 671 /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ 672 if ((qp->s_flags & QIB_S_RESP_PENDING) || qp->s_rdma_ack_cnt) 673 goto queue_ack; 674 675 /* Construct the header with s_lock held so APM doesn't change it. */ 676 ohdr = &hdr.u.oth; 677 lrh0 = QIB_LRH_BTH; 678 /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ 679 hwords = 6; 680 if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { 681 hwords += qib_make_grh(ibp, &hdr.u.l.grh, 682 &qp->remote_ah_attr.grh, hwords, 0); 683 ohdr = &hdr.u.l.oth; 684 lrh0 = QIB_LRH_GRH; 685 } 686 /* read pkey_index w/o lock (its atomic) */ 687 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); 688 if (qp->s_mig_state == IB_MIG_MIGRATED) 689 bth0 |= IB_BTH_MIG_REQ; 690 if (qp->r_nak_state) 691 ohdr->u.aeth = cpu_to_be32((qp->r_msn & QIB_MSN_MASK) | 692 (qp->r_nak_state << 693 QIB_AETH_CREDIT_SHIFT)); 694 else 695 ohdr->u.aeth = qib_compute_aeth(qp); 696 lrh0 |= ibp->sl_to_vl[qp->remote_ah_attr.sl] << 12 | 697 qp->remote_ah_attr.sl << 4; 698 hdr.lrh[0] = cpu_to_be16(lrh0); 699 hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); 700 hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); 701 hdr.lrh[3] = cpu_to_be16(ppd->lid | qp->remote_ah_attr.src_path_bits); 702 ohdr->bth[0] = cpu_to_be32(bth0); 703 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 704 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); 705 706 spin_unlock_irqrestore(&qp->s_lock, flags); 707 708 /* Don't try to send ACKs if the link isn't ACTIVE */ 709 if (!(ppd->lflags & QIBL_LINKACTIVE)) 710 goto done; 711 712 control = dd->f_setpbc_control(ppd, hwords + SIZE_OF_CRC, 713 qp->s_srate, lrh0 >> 12); 714 /* length is + 1 for the control dword */ 715 pbc = ((u64) control << 32) | (hwords + 1); 716 717 piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn); 718 if (!piobuf) { 719 /* 720 * We are out of PIO buffers at the moment. 721 * Pass responsibility for sending the ACK to the 722 * send tasklet so that when a PIO buffer becomes 723 * available, the ACK is sent ahead of other outgoing 724 * packets. 725 */ 726 spin_lock_irqsave(&qp->s_lock, flags); 727 goto queue_ack; 728 } 729 730 /* 731 * Write the pbc. 732 * We have to flush after the PBC for correctness 733 * on some cpus or WC buffer can be written out of order. 734 */ 735 writeq(pbc, piobuf); 736 737 if (dd->flags & QIB_PIO_FLUSH_WC) { 738 u32 *hdrp = (u32 *) &hdr; 739 740 qib_flush_wc(); 741 qib_pio_copy(piobuf + 2, hdrp, hwords - 1); 742 qib_flush_wc(); 743 __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1); 744 } else 745 qib_pio_copy(piobuf + 2, (u32 *) &hdr, hwords); 746 747 if (dd->flags & QIB_USE_SPCL_TRIG) { 748 u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023; 749 750 qib_flush_wc(); 751 __raw_writel(0xaebecede, piobuf + spcl_off); 752 } 753 754 qib_flush_wc(); 755 qib_sendbuf_done(dd, pbufn); 756 757 this_cpu_inc(ibp->pmastats->n_unicast_xmit); 758 goto done; 759 760 queue_ack: 761 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 762 ibp->n_rc_qacks++; 763 qp->s_flags |= QIB_S_ACK_PENDING | QIB_S_RESP_PENDING; 764 qp->s_nak_state = qp->r_nak_state; 765 qp->s_ack_psn = qp->r_ack_psn; 766 767 /* Schedule the send tasklet. */ 768 qib_schedule_send(qp); 769 } 770 unlock: 771 spin_unlock_irqrestore(&qp->s_lock, flags); 772 done: 773 return; 774 } 775 776 /** 777 * reset_psn - reset the QP state to send starting from PSN 778 * @qp: the QP 779 * @psn: the packet sequence number to restart at 780 * 781 * This is called from qib_rc_rcv() to process an incoming RC ACK 782 * for the given QP. 783 * Called at interrupt level with the QP s_lock held. 784 */ 785 static void reset_psn(struct qib_qp *qp, u32 psn) 786 { 787 u32 n = qp->s_acked; 788 struct qib_swqe *wqe = get_swqe_ptr(qp, n); 789 u32 opcode; 790 791 qp->s_cur = n; 792 793 /* 794 * If we are starting the request from the beginning, 795 * let the normal send code handle initialization. 796 */ 797 if (qib_cmp24(psn, wqe->psn) <= 0) { 798 qp->s_state = OP(SEND_LAST); 799 goto done; 800 } 801 802 /* Find the work request opcode corresponding to the given PSN. */ 803 opcode = wqe->wr.opcode; 804 for (;;) { 805 int diff; 806 807 if (++n == qp->s_size) 808 n = 0; 809 if (n == qp->s_tail) 810 break; 811 wqe = get_swqe_ptr(qp, n); 812 diff = qib_cmp24(psn, wqe->psn); 813 if (diff < 0) 814 break; 815 qp->s_cur = n; 816 /* 817 * If we are starting the request from the beginning, 818 * let the normal send code handle initialization. 819 */ 820 if (diff == 0) { 821 qp->s_state = OP(SEND_LAST); 822 goto done; 823 } 824 opcode = wqe->wr.opcode; 825 } 826 827 /* 828 * Set the state to restart in the middle of a request. 829 * Don't change the s_sge, s_cur_sge, or s_cur_size. 830 * See qib_make_rc_req(). 831 */ 832 switch (opcode) { 833 case IB_WR_SEND: 834 case IB_WR_SEND_WITH_IMM: 835 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); 836 break; 837 838 case IB_WR_RDMA_WRITE: 839 case IB_WR_RDMA_WRITE_WITH_IMM: 840 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); 841 break; 842 843 case IB_WR_RDMA_READ: 844 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); 845 break; 846 847 default: 848 /* 849 * This case shouldn't happen since its only 850 * one PSN per req. 851 */ 852 qp->s_state = OP(SEND_LAST); 853 } 854 done: 855 qp->s_psn = psn; 856 /* 857 * Set QIB_S_WAIT_PSN as qib_rc_complete() may start the timer 858 * asynchronously before the send tasklet can get scheduled. 859 * Doing it in qib_make_rc_req() is too late. 860 */ 861 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && 862 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) 863 qp->s_flags |= QIB_S_WAIT_PSN; 864 } 865 866 /* 867 * Back up requester to resend the last un-ACKed request. 868 * The QP r_lock and s_lock should be held and interrupts disabled. 869 */ 870 static void qib_restart_rc(struct qib_qp *qp, u32 psn, int wait) 871 { 872 struct qib_swqe *wqe = get_swqe_ptr(qp, qp->s_acked); 873 struct qib_ibport *ibp; 874 875 if (qp->s_retry == 0) { 876 if (qp->s_mig_state == IB_MIG_ARMED) { 877 qib_migrate_qp(qp); 878 qp->s_retry = qp->s_retry_cnt; 879 } else if (qp->s_last == qp->s_acked) { 880 qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); 881 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 882 return; 883 } else /* XXX need to handle delayed completion */ 884 return; 885 } else 886 qp->s_retry--; 887 888 ibp = to_iport(qp->ibqp.device, qp->port_num); 889 if (wqe->wr.opcode == IB_WR_RDMA_READ) 890 ibp->n_rc_resends++; 891 else 892 ibp->n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; 893 894 qp->s_flags &= ~(QIB_S_WAIT_FENCE | QIB_S_WAIT_RDMAR | 895 QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_PSN | 896 QIB_S_WAIT_ACK); 897 if (wait) 898 qp->s_flags |= QIB_S_SEND_ONE; 899 reset_psn(qp, psn); 900 } 901 902 /* 903 * This is called from s_timer for missing responses. 904 */ 905 static void rc_timeout(unsigned long arg) 906 { 907 struct qib_qp *qp = (struct qib_qp *)arg; 908 struct qib_ibport *ibp; 909 unsigned long flags; 910 911 spin_lock_irqsave(&qp->r_lock, flags); 912 spin_lock(&qp->s_lock); 913 if (qp->s_flags & QIB_S_TIMER) { 914 ibp = to_iport(qp->ibqp.device, qp->port_num); 915 ibp->n_rc_timeouts++; 916 qp->s_flags &= ~QIB_S_TIMER; 917 del_timer(&qp->s_timer); 918 qib_restart_rc(qp, qp->s_last_psn + 1, 1); 919 qib_schedule_send(qp); 920 } 921 spin_unlock(&qp->s_lock); 922 spin_unlock_irqrestore(&qp->r_lock, flags); 923 } 924 925 /* 926 * This is called from s_timer for RNR timeouts. 927 */ 928 void qib_rc_rnr_retry(unsigned long arg) 929 { 930 struct qib_qp *qp = (struct qib_qp *)arg; 931 unsigned long flags; 932 933 spin_lock_irqsave(&qp->s_lock, flags); 934 if (qp->s_flags & QIB_S_WAIT_RNR) { 935 qp->s_flags &= ~QIB_S_WAIT_RNR; 936 del_timer(&qp->s_timer); 937 qib_schedule_send(qp); 938 } 939 spin_unlock_irqrestore(&qp->s_lock, flags); 940 } 941 942 /* 943 * Set qp->s_sending_psn to the next PSN after the given one. 944 * This would be psn+1 except when RDMA reads are present. 945 */ 946 static void reset_sending_psn(struct qib_qp *qp, u32 psn) 947 { 948 struct qib_swqe *wqe; 949 u32 n = qp->s_last; 950 951 /* Find the work request corresponding to the given PSN. */ 952 for (;;) { 953 wqe = get_swqe_ptr(qp, n); 954 if (qib_cmp24(psn, wqe->lpsn) <= 0) { 955 if (wqe->wr.opcode == IB_WR_RDMA_READ) 956 qp->s_sending_psn = wqe->lpsn + 1; 957 else 958 qp->s_sending_psn = psn + 1; 959 break; 960 } 961 if (++n == qp->s_size) 962 n = 0; 963 if (n == qp->s_tail) 964 break; 965 } 966 } 967 968 /* 969 * This should be called with the QP s_lock held and interrupts disabled. 970 */ 971 void qib_rc_send_complete(struct qib_qp *qp, struct qib_ib_header *hdr) 972 { 973 struct qib_other_headers *ohdr; 974 struct qib_swqe *wqe; 975 struct ib_wc wc; 976 unsigned i; 977 u32 opcode; 978 u32 psn; 979 980 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_OR_FLUSH_SEND)) 981 return; 982 983 /* Find out where the BTH is */ 984 if ((be16_to_cpu(hdr->lrh[0]) & 3) == QIB_LRH_BTH) 985 ohdr = &hdr->u.oth; 986 else 987 ohdr = &hdr->u.l.oth; 988 989 opcode = be32_to_cpu(ohdr->bth[0]) >> 24; 990 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 991 opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 992 WARN_ON(!qp->s_rdma_ack_cnt); 993 qp->s_rdma_ack_cnt--; 994 return; 995 } 996 997 psn = be32_to_cpu(ohdr->bth[2]); 998 reset_sending_psn(qp, psn); 999 1000 /* 1001 * Start timer after a packet requesting an ACK has been sent and 1002 * there are still requests that haven't been acked. 1003 */ 1004 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && 1005 !(qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR | QIB_S_WAIT_PSN)) && 1006 (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 1007 start_timer(qp); 1008 1009 while (qp->s_last != qp->s_acked) { 1010 wqe = get_swqe_ptr(qp, qp->s_last); 1011 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && 1012 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) 1013 break; 1014 for (i = 0; i < wqe->wr.num_sge; i++) { 1015 struct qib_sge *sge = &wqe->sg_list[i]; 1016 1017 qib_put_mr(sge->mr); 1018 } 1019 /* Post a send completion queue entry if requested. */ 1020 if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) || 1021 (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 1022 memset(&wc, 0, sizeof(wc)); 1023 wc.wr_id = wqe->wr.wr_id; 1024 wc.status = IB_WC_SUCCESS; 1025 wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode]; 1026 wc.byte_len = wqe->length; 1027 wc.qp = &qp->ibqp; 1028 qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 1029 } 1030 if (++qp->s_last >= qp->s_size) 1031 qp->s_last = 0; 1032 } 1033 /* 1034 * If we were waiting for sends to complete before resending, 1035 * and they are now complete, restart sending. 1036 */ 1037 if (qp->s_flags & QIB_S_WAIT_PSN && 1038 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 1039 qp->s_flags &= ~QIB_S_WAIT_PSN; 1040 qp->s_sending_psn = qp->s_psn; 1041 qp->s_sending_hpsn = qp->s_psn - 1; 1042 qib_schedule_send(qp); 1043 } 1044 } 1045 1046 static inline void update_last_psn(struct qib_qp *qp, u32 psn) 1047 { 1048 qp->s_last_psn = psn; 1049 } 1050 1051 /* 1052 * Generate a SWQE completion. 1053 * This is similar to qib_send_complete but has to check to be sure 1054 * that the SGEs are not being referenced if the SWQE is being resent. 1055 */ 1056 static struct qib_swqe *do_rc_completion(struct qib_qp *qp, 1057 struct qib_swqe *wqe, 1058 struct qib_ibport *ibp) 1059 { 1060 struct ib_wc wc; 1061 unsigned i; 1062 1063 /* 1064 * Don't decrement refcount and don't generate a 1065 * completion if the SWQE is being resent until the send 1066 * is finished. 1067 */ 1068 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || 1069 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 1070 for (i = 0; i < wqe->wr.num_sge; i++) { 1071 struct qib_sge *sge = &wqe->sg_list[i]; 1072 1073 qib_put_mr(sge->mr); 1074 } 1075 /* Post a send completion queue entry if requested. */ 1076 if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) || 1077 (wqe->wr.send_flags & IB_SEND_SIGNALED)) { 1078 memset(&wc, 0, sizeof(wc)); 1079 wc.wr_id = wqe->wr.wr_id; 1080 wc.status = IB_WC_SUCCESS; 1081 wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode]; 1082 wc.byte_len = wqe->length; 1083 wc.qp = &qp->ibqp; 1084 qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); 1085 } 1086 if (++qp->s_last >= qp->s_size) 1087 qp->s_last = 0; 1088 } else 1089 ibp->n_rc_delayed_comp++; 1090 1091 qp->s_retry = qp->s_retry_cnt; 1092 update_last_psn(qp, wqe->lpsn); 1093 1094 /* 1095 * If we are completing a request which is in the process of 1096 * being resent, we can stop resending it since we know the 1097 * responder has already seen it. 1098 */ 1099 if (qp->s_acked == qp->s_cur) { 1100 if (++qp->s_cur >= qp->s_size) 1101 qp->s_cur = 0; 1102 qp->s_acked = qp->s_cur; 1103 wqe = get_swqe_ptr(qp, qp->s_cur); 1104 if (qp->s_acked != qp->s_tail) { 1105 qp->s_state = OP(SEND_LAST); 1106 qp->s_psn = wqe->psn; 1107 } 1108 } else { 1109 if (++qp->s_acked >= qp->s_size) 1110 qp->s_acked = 0; 1111 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) 1112 qp->s_draining = 0; 1113 wqe = get_swqe_ptr(qp, qp->s_acked); 1114 } 1115 return wqe; 1116 } 1117 1118 /** 1119 * do_rc_ack - process an incoming RC ACK 1120 * @qp: the QP the ACK came in on 1121 * @psn: the packet sequence number of the ACK 1122 * @opcode: the opcode of the request that resulted in the ACK 1123 * 1124 * This is called from qib_rc_rcv_resp() to process an incoming RC ACK 1125 * for the given QP. 1126 * Called at interrupt level with the QP s_lock held. 1127 * Returns 1 if OK, 0 if current operation should be aborted (NAK). 1128 */ 1129 static int do_rc_ack(struct qib_qp *qp, u32 aeth, u32 psn, int opcode, 1130 u64 val, struct qib_ctxtdata *rcd) 1131 { 1132 struct qib_ibport *ibp; 1133 enum ib_wc_status status; 1134 struct qib_swqe *wqe; 1135 int ret = 0; 1136 u32 ack_psn; 1137 int diff; 1138 1139 /* Remove QP from retry timer */ 1140 if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) { 1141 qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR); 1142 del_timer(&qp->s_timer); 1143 } 1144 1145 /* 1146 * Note that NAKs implicitly ACK outstanding SEND and RDMA write 1147 * requests and implicitly NAK RDMA read and atomic requests issued 1148 * before the NAK'ed request. The MSN won't include the NAK'ed 1149 * request but will include an ACK'ed request(s). 1150 */ 1151 ack_psn = psn; 1152 if (aeth >> 29) 1153 ack_psn--; 1154 wqe = get_swqe_ptr(qp, qp->s_acked); 1155 ibp = to_iport(qp->ibqp.device, qp->port_num); 1156 1157 /* 1158 * The MSN might be for a later WQE than the PSN indicates so 1159 * only complete WQEs that the PSN finishes. 1160 */ 1161 while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) { 1162 /* 1163 * RDMA_READ_RESPONSE_ONLY is a special case since 1164 * we want to generate completion events for everything 1165 * before the RDMA read, copy the data, then generate 1166 * the completion for the read. 1167 */ 1168 if (wqe->wr.opcode == IB_WR_RDMA_READ && 1169 opcode == OP(RDMA_READ_RESPONSE_ONLY) && 1170 diff == 0) { 1171 ret = 1; 1172 goto bail; 1173 } 1174 /* 1175 * If this request is a RDMA read or atomic, and the ACK is 1176 * for a later operation, this ACK NAKs the RDMA read or 1177 * atomic. In other words, only a RDMA_READ_LAST or ONLY 1178 * can ACK a RDMA read and likewise for atomic ops. Note 1179 * that the NAK case can only happen if relaxed ordering is 1180 * used and requests are sent after an RDMA read or atomic 1181 * is sent but before the response is received. 1182 */ 1183 if ((wqe->wr.opcode == IB_WR_RDMA_READ && 1184 (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) || 1185 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1186 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && 1187 (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) { 1188 /* Retry this request. */ 1189 if (!(qp->r_flags & QIB_R_RDMAR_SEQ)) { 1190 qp->r_flags |= QIB_R_RDMAR_SEQ; 1191 qib_restart_rc(qp, qp->s_last_psn + 1, 0); 1192 if (list_empty(&qp->rspwait)) { 1193 qp->r_flags |= QIB_R_RSP_SEND; 1194 atomic_inc(&qp->refcount); 1195 list_add_tail(&qp->rspwait, 1196 &rcd->qp_wait_list); 1197 } 1198 } 1199 /* 1200 * No need to process the ACK/NAK since we are 1201 * restarting an earlier request. 1202 */ 1203 goto bail; 1204 } 1205 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1206 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 1207 u64 *vaddr = wqe->sg_list[0].vaddr; 1208 *vaddr = val; 1209 } 1210 if (qp->s_num_rd_atomic && 1211 (wqe->wr.opcode == IB_WR_RDMA_READ || 1212 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1213 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { 1214 qp->s_num_rd_atomic--; 1215 /* Restart sending task if fence is complete */ 1216 if ((qp->s_flags & QIB_S_WAIT_FENCE) && 1217 !qp->s_num_rd_atomic) { 1218 qp->s_flags &= ~(QIB_S_WAIT_FENCE | 1219 QIB_S_WAIT_ACK); 1220 qib_schedule_send(qp); 1221 } else if (qp->s_flags & QIB_S_WAIT_RDMAR) { 1222 qp->s_flags &= ~(QIB_S_WAIT_RDMAR | 1223 QIB_S_WAIT_ACK); 1224 qib_schedule_send(qp); 1225 } 1226 } 1227 wqe = do_rc_completion(qp, wqe, ibp); 1228 if (qp->s_acked == qp->s_tail) 1229 break; 1230 } 1231 1232 switch (aeth >> 29) { 1233 case 0: /* ACK */ 1234 ibp->n_rc_acks++; 1235 if (qp->s_acked != qp->s_tail) { 1236 /* 1237 * We are expecting more ACKs so 1238 * reset the retransmit timer. 1239 */ 1240 start_timer(qp); 1241 /* 1242 * We can stop resending the earlier packets and 1243 * continue with the next packet the receiver wants. 1244 */ 1245 if (qib_cmp24(qp->s_psn, psn) <= 0) 1246 reset_psn(qp, psn + 1); 1247 } else if (qib_cmp24(qp->s_psn, psn) <= 0) { 1248 qp->s_state = OP(SEND_LAST); 1249 qp->s_psn = psn + 1; 1250 } 1251 if (qp->s_flags & QIB_S_WAIT_ACK) { 1252 qp->s_flags &= ~QIB_S_WAIT_ACK; 1253 qib_schedule_send(qp); 1254 } 1255 qib_get_credit(qp, aeth); 1256 qp->s_rnr_retry = qp->s_rnr_retry_cnt; 1257 qp->s_retry = qp->s_retry_cnt; 1258 update_last_psn(qp, psn); 1259 ret = 1; 1260 goto bail; 1261 1262 case 1: /* RNR NAK */ 1263 ibp->n_rnr_naks++; 1264 if (qp->s_acked == qp->s_tail) 1265 goto bail; 1266 if (qp->s_flags & QIB_S_WAIT_RNR) 1267 goto bail; 1268 if (qp->s_rnr_retry == 0) { 1269 status = IB_WC_RNR_RETRY_EXC_ERR; 1270 goto class_b; 1271 } 1272 if (qp->s_rnr_retry_cnt < 7) 1273 qp->s_rnr_retry--; 1274 1275 /* The last valid PSN is the previous PSN. */ 1276 update_last_psn(qp, psn - 1); 1277 1278 ibp->n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; 1279 1280 reset_psn(qp, psn); 1281 1282 qp->s_flags &= ~(QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_ACK); 1283 qp->s_flags |= QIB_S_WAIT_RNR; 1284 qp->s_timer.function = qib_rc_rnr_retry; 1285 qp->s_timer.expires = jiffies + usecs_to_jiffies( 1286 ib_qib_rnr_table[(aeth >> QIB_AETH_CREDIT_SHIFT) & 1287 QIB_AETH_CREDIT_MASK]); 1288 add_timer(&qp->s_timer); 1289 goto bail; 1290 1291 case 3: /* NAK */ 1292 if (qp->s_acked == qp->s_tail) 1293 goto bail; 1294 /* The last valid PSN is the previous PSN. */ 1295 update_last_psn(qp, psn - 1); 1296 switch ((aeth >> QIB_AETH_CREDIT_SHIFT) & 1297 QIB_AETH_CREDIT_MASK) { 1298 case 0: /* PSN sequence error */ 1299 ibp->n_seq_naks++; 1300 /* 1301 * Back up to the responder's expected PSN. 1302 * Note that we might get a NAK in the middle of an 1303 * RDMA READ response which terminates the RDMA 1304 * READ. 1305 */ 1306 qib_restart_rc(qp, psn, 0); 1307 qib_schedule_send(qp); 1308 break; 1309 1310 case 1: /* Invalid Request */ 1311 status = IB_WC_REM_INV_REQ_ERR; 1312 ibp->n_other_naks++; 1313 goto class_b; 1314 1315 case 2: /* Remote Access Error */ 1316 status = IB_WC_REM_ACCESS_ERR; 1317 ibp->n_other_naks++; 1318 goto class_b; 1319 1320 case 3: /* Remote Operation Error */ 1321 status = IB_WC_REM_OP_ERR; 1322 ibp->n_other_naks++; 1323 class_b: 1324 if (qp->s_last == qp->s_acked) { 1325 qib_send_complete(qp, wqe, status); 1326 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1327 } 1328 break; 1329 1330 default: 1331 /* Ignore other reserved NAK error codes */ 1332 goto reserved; 1333 } 1334 qp->s_retry = qp->s_retry_cnt; 1335 qp->s_rnr_retry = qp->s_rnr_retry_cnt; 1336 goto bail; 1337 1338 default: /* 2: reserved */ 1339 reserved: 1340 /* Ignore reserved NAK codes. */ 1341 goto bail; 1342 } 1343 1344 bail: 1345 return ret; 1346 } 1347 1348 /* 1349 * We have seen an out of sequence RDMA read middle or last packet. 1350 * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE. 1351 */ 1352 static void rdma_seq_err(struct qib_qp *qp, struct qib_ibport *ibp, u32 psn, 1353 struct qib_ctxtdata *rcd) 1354 { 1355 struct qib_swqe *wqe; 1356 1357 /* Remove QP from retry timer */ 1358 if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) { 1359 qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR); 1360 del_timer(&qp->s_timer); 1361 } 1362 1363 wqe = get_swqe_ptr(qp, qp->s_acked); 1364 1365 while (qib_cmp24(psn, wqe->lpsn) > 0) { 1366 if (wqe->wr.opcode == IB_WR_RDMA_READ || 1367 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1368 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 1369 break; 1370 wqe = do_rc_completion(qp, wqe, ibp); 1371 } 1372 1373 ibp->n_rdma_seq++; 1374 qp->r_flags |= QIB_R_RDMAR_SEQ; 1375 qib_restart_rc(qp, qp->s_last_psn + 1, 0); 1376 if (list_empty(&qp->rspwait)) { 1377 qp->r_flags |= QIB_R_RSP_SEND; 1378 atomic_inc(&qp->refcount); 1379 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 1380 } 1381 } 1382 1383 /** 1384 * qib_rc_rcv_resp - process an incoming RC response packet 1385 * @ibp: the port this packet came in on 1386 * @ohdr: the other headers for this packet 1387 * @data: the packet data 1388 * @tlen: the packet length 1389 * @qp: the QP for this packet 1390 * @opcode: the opcode for this packet 1391 * @psn: the packet sequence number for this packet 1392 * @hdrsize: the header length 1393 * @pmtu: the path MTU 1394 * 1395 * This is called from qib_rc_rcv() to process an incoming RC response 1396 * packet for the given QP. 1397 * Called at interrupt level. 1398 */ 1399 static void qib_rc_rcv_resp(struct qib_ibport *ibp, 1400 struct qib_other_headers *ohdr, 1401 void *data, u32 tlen, 1402 struct qib_qp *qp, 1403 u32 opcode, 1404 u32 psn, u32 hdrsize, u32 pmtu, 1405 struct qib_ctxtdata *rcd) 1406 { 1407 struct qib_swqe *wqe; 1408 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1409 enum ib_wc_status status; 1410 unsigned long flags; 1411 int diff; 1412 u32 pad; 1413 u32 aeth; 1414 u64 val; 1415 1416 if (opcode != OP(RDMA_READ_RESPONSE_MIDDLE)) { 1417 /* 1418 * If ACK'd PSN on SDMA busy list try to make progress to 1419 * reclaim SDMA credits. 1420 */ 1421 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && 1422 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { 1423 1424 /* 1425 * If send tasklet not running attempt to progress 1426 * SDMA queue. 1427 */ 1428 if (!(qp->s_flags & QIB_S_BUSY)) { 1429 /* Acquire SDMA Lock */ 1430 spin_lock_irqsave(&ppd->sdma_lock, flags); 1431 /* Invoke sdma make progress */ 1432 qib_sdma_make_progress(ppd); 1433 /* Release SDMA Lock */ 1434 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 1435 } 1436 } 1437 } 1438 1439 spin_lock_irqsave(&qp->s_lock, flags); 1440 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) 1441 goto ack_done; 1442 1443 /* Ignore invalid responses. */ 1444 if (qib_cmp24(psn, qp->s_next_psn) >= 0) 1445 goto ack_done; 1446 1447 /* Ignore duplicate responses. */ 1448 diff = qib_cmp24(psn, qp->s_last_psn); 1449 if (unlikely(diff <= 0)) { 1450 /* Update credits for "ghost" ACKs */ 1451 if (diff == 0 && opcode == OP(ACKNOWLEDGE)) { 1452 aeth = be32_to_cpu(ohdr->u.aeth); 1453 if ((aeth >> 29) == 0) 1454 qib_get_credit(qp, aeth); 1455 } 1456 goto ack_done; 1457 } 1458 1459 /* 1460 * Skip everything other than the PSN we expect, if we are waiting 1461 * for a reply to a restarted RDMA read or atomic op. 1462 */ 1463 if (qp->r_flags & QIB_R_RDMAR_SEQ) { 1464 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) 1465 goto ack_done; 1466 qp->r_flags &= ~QIB_R_RDMAR_SEQ; 1467 } 1468 1469 if (unlikely(qp->s_acked == qp->s_tail)) 1470 goto ack_done; 1471 wqe = get_swqe_ptr(qp, qp->s_acked); 1472 status = IB_WC_SUCCESS; 1473 1474 switch (opcode) { 1475 case OP(ACKNOWLEDGE): 1476 case OP(ATOMIC_ACKNOWLEDGE): 1477 case OP(RDMA_READ_RESPONSE_FIRST): 1478 aeth = be32_to_cpu(ohdr->u.aeth); 1479 if (opcode == OP(ATOMIC_ACKNOWLEDGE)) { 1480 __be32 *p = ohdr->u.at.atomic_ack_eth; 1481 1482 val = ((u64) be32_to_cpu(p[0]) << 32) | 1483 be32_to_cpu(p[1]); 1484 } else 1485 val = 0; 1486 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || 1487 opcode != OP(RDMA_READ_RESPONSE_FIRST)) 1488 goto ack_done; 1489 hdrsize += 4; 1490 wqe = get_swqe_ptr(qp, qp->s_acked); 1491 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1492 goto ack_op_err; 1493 /* 1494 * If this is a response to a resent RDMA read, we 1495 * have to be careful to copy the data to the right 1496 * location. 1497 */ 1498 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 1499 wqe, psn, pmtu); 1500 goto read_middle; 1501 1502 case OP(RDMA_READ_RESPONSE_MIDDLE): 1503 /* no AETH, no ACK */ 1504 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) 1505 goto ack_seq_err; 1506 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1507 goto ack_op_err; 1508 read_middle: 1509 if (unlikely(tlen != (hdrsize + pmtu + 4))) 1510 goto ack_len_err; 1511 if (unlikely(pmtu >= qp->s_rdma_read_len)) 1512 goto ack_len_err; 1513 1514 /* 1515 * We got a response so update the timeout. 1516 * 4.096 usec. * (1 << qp->timeout) 1517 */ 1518 qp->s_flags |= QIB_S_TIMER; 1519 mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies); 1520 if (qp->s_flags & QIB_S_WAIT_ACK) { 1521 qp->s_flags &= ~QIB_S_WAIT_ACK; 1522 qib_schedule_send(qp); 1523 } 1524 1525 if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE)) 1526 qp->s_retry = qp->s_retry_cnt; 1527 1528 /* 1529 * Update the RDMA receive state but do the copy w/o 1530 * holding the locks and blocking interrupts. 1531 */ 1532 qp->s_rdma_read_len -= pmtu; 1533 update_last_psn(qp, psn); 1534 spin_unlock_irqrestore(&qp->s_lock, flags); 1535 qib_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0); 1536 goto bail; 1537 1538 case OP(RDMA_READ_RESPONSE_ONLY): 1539 aeth = be32_to_cpu(ohdr->u.aeth); 1540 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) 1541 goto ack_done; 1542 /* Get the number of bytes the message was padded by. */ 1543 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1544 /* 1545 * Check that the data size is >= 0 && <= pmtu. 1546 * Remember to account for the AETH header (4) and 1547 * ICRC (4). 1548 */ 1549 if (unlikely(tlen < (hdrsize + pad + 8))) 1550 goto ack_len_err; 1551 /* 1552 * If this is a response to a resent RDMA read, we 1553 * have to be careful to copy the data to the right 1554 * location. 1555 */ 1556 wqe = get_swqe_ptr(qp, qp->s_acked); 1557 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 1558 wqe, psn, pmtu); 1559 goto read_last; 1560 1561 case OP(RDMA_READ_RESPONSE_LAST): 1562 /* ACKs READ req. */ 1563 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) 1564 goto ack_seq_err; 1565 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1566 goto ack_op_err; 1567 /* Get the number of bytes the message was padded by. */ 1568 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1569 /* 1570 * Check that the data size is >= 1 && <= pmtu. 1571 * Remember to account for the AETH header (4) and 1572 * ICRC (4). 1573 */ 1574 if (unlikely(tlen <= (hdrsize + pad + 8))) 1575 goto ack_len_err; 1576 read_last: 1577 tlen -= hdrsize + pad + 8; 1578 if (unlikely(tlen != qp->s_rdma_read_len)) 1579 goto ack_len_err; 1580 aeth = be32_to_cpu(ohdr->u.aeth); 1581 qib_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0); 1582 WARN_ON(qp->s_rdma_read_sge.num_sge); 1583 (void) do_rc_ack(qp, aeth, psn, 1584 OP(RDMA_READ_RESPONSE_LAST), 0, rcd); 1585 goto ack_done; 1586 } 1587 1588 ack_op_err: 1589 status = IB_WC_LOC_QP_OP_ERR; 1590 goto ack_err; 1591 1592 ack_seq_err: 1593 rdma_seq_err(qp, ibp, psn, rcd); 1594 goto ack_done; 1595 1596 ack_len_err: 1597 status = IB_WC_LOC_LEN_ERR; 1598 ack_err: 1599 if (qp->s_last == qp->s_acked) { 1600 qib_send_complete(qp, wqe, status); 1601 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1602 } 1603 ack_done: 1604 spin_unlock_irqrestore(&qp->s_lock, flags); 1605 bail: 1606 return; 1607 } 1608 1609 /** 1610 * qib_rc_rcv_error - process an incoming duplicate or error RC packet 1611 * @ohdr: the other headers for this packet 1612 * @data: the packet data 1613 * @qp: the QP for this packet 1614 * @opcode: the opcode for this packet 1615 * @psn: the packet sequence number for this packet 1616 * @diff: the difference between the PSN and the expected PSN 1617 * 1618 * This is called from qib_rc_rcv() to process an unexpected 1619 * incoming RC packet for the given QP. 1620 * Called at interrupt level. 1621 * Return 1 if no more processing is needed; otherwise return 0 to 1622 * schedule a response to be sent. 1623 */ 1624 static int qib_rc_rcv_error(struct qib_other_headers *ohdr, 1625 void *data, 1626 struct qib_qp *qp, 1627 u32 opcode, 1628 u32 psn, 1629 int diff, 1630 struct qib_ctxtdata *rcd) 1631 { 1632 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 1633 struct qib_ack_entry *e; 1634 unsigned long flags; 1635 u8 i, prev; 1636 int old_req; 1637 1638 if (diff > 0) { 1639 /* 1640 * Packet sequence error. 1641 * A NAK will ACK earlier sends and RDMA writes. 1642 * Don't queue the NAK if we already sent one. 1643 */ 1644 if (!qp->r_nak_state) { 1645 ibp->n_rc_seqnak++; 1646 qp->r_nak_state = IB_NAK_PSN_ERROR; 1647 /* Use the expected PSN. */ 1648 qp->r_ack_psn = qp->r_psn; 1649 /* 1650 * Wait to send the sequence NAK until all packets 1651 * in the receive queue have been processed. 1652 * Otherwise, we end up propagating congestion. 1653 */ 1654 if (list_empty(&qp->rspwait)) { 1655 qp->r_flags |= QIB_R_RSP_NAK; 1656 atomic_inc(&qp->refcount); 1657 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 1658 } 1659 } 1660 goto done; 1661 } 1662 1663 /* 1664 * Handle a duplicate request. Don't re-execute SEND, RDMA 1665 * write or atomic op. Don't NAK errors, just silently drop 1666 * the duplicate request. Note that r_sge, r_len, and 1667 * r_rcv_len may be in use so don't modify them. 1668 * 1669 * We are supposed to ACK the earliest duplicate PSN but we 1670 * can coalesce an outstanding duplicate ACK. We have to 1671 * send the earliest so that RDMA reads can be restarted at 1672 * the requester's expected PSN. 1673 * 1674 * First, find where this duplicate PSN falls within the 1675 * ACKs previously sent. 1676 * old_req is true if there is an older response that is scheduled 1677 * to be sent before sending this one. 1678 */ 1679 e = NULL; 1680 old_req = 1; 1681 ibp->n_rc_dupreq++; 1682 1683 spin_lock_irqsave(&qp->s_lock, flags); 1684 1685 for (i = qp->r_head_ack_queue; ; i = prev) { 1686 if (i == qp->s_tail_ack_queue) 1687 old_req = 0; 1688 if (i) 1689 prev = i - 1; 1690 else 1691 prev = QIB_MAX_RDMA_ATOMIC; 1692 if (prev == qp->r_head_ack_queue) { 1693 e = NULL; 1694 break; 1695 } 1696 e = &qp->s_ack_queue[prev]; 1697 if (!e->opcode) { 1698 e = NULL; 1699 break; 1700 } 1701 if (qib_cmp24(psn, e->psn) >= 0) { 1702 if (prev == qp->s_tail_ack_queue && 1703 qib_cmp24(psn, e->lpsn) <= 0) 1704 old_req = 0; 1705 break; 1706 } 1707 } 1708 switch (opcode) { 1709 case OP(RDMA_READ_REQUEST): { 1710 struct ib_reth *reth; 1711 u32 offset; 1712 u32 len; 1713 1714 /* 1715 * If we didn't find the RDMA read request in the ack queue, 1716 * we can ignore this request. 1717 */ 1718 if (!e || e->opcode != OP(RDMA_READ_REQUEST)) 1719 goto unlock_done; 1720 /* RETH comes after BTH */ 1721 reth = &ohdr->u.rc.reth; 1722 /* 1723 * Address range must be a subset of the original 1724 * request and start on pmtu boundaries. 1725 * We reuse the old ack_queue slot since the requester 1726 * should not back up and request an earlier PSN for the 1727 * same request. 1728 */ 1729 offset = ((psn - e->psn) & QIB_PSN_MASK) * 1730 qp->pmtu; 1731 len = be32_to_cpu(reth->length); 1732 if (unlikely(offset + len != e->rdma_sge.sge_length)) 1733 goto unlock_done; 1734 if (e->rdma_sge.mr) { 1735 qib_put_mr(e->rdma_sge.mr); 1736 e->rdma_sge.mr = NULL; 1737 } 1738 if (len != 0) { 1739 u32 rkey = be32_to_cpu(reth->rkey); 1740 u64 vaddr = be64_to_cpu(reth->vaddr); 1741 int ok; 1742 1743 ok = qib_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, 1744 IB_ACCESS_REMOTE_READ); 1745 if (unlikely(!ok)) 1746 goto unlock_done; 1747 } else { 1748 e->rdma_sge.vaddr = NULL; 1749 e->rdma_sge.length = 0; 1750 e->rdma_sge.sge_length = 0; 1751 } 1752 e->psn = psn; 1753 if (old_req) 1754 goto unlock_done; 1755 qp->s_tail_ack_queue = prev; 1756 break; 1757 } 1758 1759 case OP(COMPARE_SWAP): 1760 case OP(FETCH_ADD): { 1761 /* 1762 * If we didn't find the atomic request in the ack queue 1763 * or the send tasklet is already backed up to send an 1764 * earlier entry, we can ignore this request. 1765 */ 1766 if (!e || e->opcode != (u8) opcode || old_req) 1767 goto unlock_done; 1768 qp->s_tail_ack_queue = prev; 1769 break; 1770 } 1771 1772 default: 1773 /* 1774 * Ignore this operation if it doesn't request an ACK 1775 * or an earlier RDMA read or atomic is going to be resent. 1776 */ 1777 if (!(psn & IB_BTH_REQ_ACK) || old_req) 1778 goto unlock_done; 1779 /* 1780 * Resend the most recent ACK if this request is 1781 * after all the previous RDMA reads and atomics. 1782 */ 1783 if (i == qp->r_head_ack_queue) { 1784 spin_unlock_irqrestore(&qp->s_lock, flags); 1785 qp->r_nak_state = 0; 1786 qp->r_ack_psn = qp->r_psn - 1; 1787 goto send_ack; 1788 } 1789 /* 1790 * Try to send a simple ACK to work around a Mellanox bug 1791 * which doesn't accept a RDMA read response or atomic 1792 * response as an ACK for earlier SENDs or RDMA writes. 1793 */ 1794 if (!(qp->s_flags & QIB_S_RESP_PENDING)) { 1795 spin_unlock_irqrestore(&qp->s_lock, flags); 1796 qp->r_nak_state = 0; 1797 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; 1798 goto send_ack; 1799 } 1800 /* 1801 * Resend the RDMA read or atomic op which 1802 * ACKs this duplicate request. 1803 */ 1804 qp->s_tail_ack_queue = i; 1805 break; 1806 } 1807 qp->s_ack_state = OP(ACKNOWLEDGE); 1808 qp->s_flags |= QIB_S_RESP_PENDING; 1809 qp->r_nak_state = 0; 1810 qib_schedule_send(qp); 1811 1812 unlock_done: 1813 spin_unlock_irqrestore(&qp->s_lock, flags); 1814 done: 1815 return 1; 1816 1817 send_ack: 1818 return 0; 1819 } 1820 1821 void qib_rc_error(struct qib_qp *qp, enum ib_wc_status err) 1822 { 1823 unsigned long flags; 1824 int lastwqe; 1825 1826 spin_lock_irqsave(&qp->s_lock, flags); 1827 lastwqe = qib_error_qp(qp, err); 1828 spin_unlock_irqrestore(&qp->s_lock, flags); 1829 1830 if (lastwqe) { 1831 struct ib_event ev; 1832 1833 ev.device = qp->ibqp.device; 1834 ev.element.qp = &qp->ibqp; 1835 ev.event = IB_EVENT_QP_LAST_WQE_REACHED; 1836 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1837 } 1838 } 1839 1840 static inline void qib_update_ack_queue(struct qib_qp *qp, unsigned n) 1841 { 1842 unsigned next; 1843 1844 next = n + 1; 1845 if (next > QIB_MAX_RDMA_ATOMIC) 1846 next = 0; 1847 qp->s_tail_ack_queue = next; 1848 qp->s_ack_state = OP(ACKNOWLEDGE); 1849 } 1850 1851 /** 1852 * qib_rc_rcv - process an incoming RC packet 1853 * @rcd: the context pointer 1854 * @hdr: the header of this packet 1855 * @has_grh: true if the header has a GRH 1856 * @data: the packet data 1857 * @tlen: the packet length 1858 * @qp: the QP for this packet 1859 * 1860 * This is called from qib_qp_rcv() to process an incoming RC packet 1861 * for the given QP. 1862 * Called at interrupt level. 1863 */ 1864 void qib_rc_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, 1865 int has_grh, void *data, u32 tlen, struct qib_qp *qp) 1866 { 1867 struct qib_ibport *ibp = &rcd->ppd->ibport_data; 1868 struct qib_other_headers *ohdr; 1869 u32 opcode; 1870 u32 hdrsize; 1871 u32 psn; 1872 u32 pad; 1873 struct ib_wc wc; 1874 u32 pmtu = qp->pmtu; 1875 int diff; 1876 struct ib_reth *reth; 1877 unsigned long flags; 1878 int ret; 1879 1880 /* Check for GRH */ 1881 if (!has_grh) { 1882 ohdr = &hdr->u.oth; 1883 hdrsize = 8 + 12; /* LRH + BTH */ 1884 } else { 1885 ohdr = &hdr->u.l.oth; 1886 hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 1887 } 1888 1889 opcode = be32_to_cpu(ohdr->bth[0]); 1890 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) 1891 return; 1892 1893 psn = be32_to_cpu(ohdr->bth[2]); 1894 opcode >>= 24; 1895 1896 /* 1897 * Process responses (ACKs) before anything else. Note that the 1898 * packet sequence number will be for something in the send work 1899 * queue rather than the expected receive packet sequence number. 1900 * In other words, this QP is the requester. 1901 */ 1902 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 1903 opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 1904 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, 1905 hdrsize, pmtu, rcd); 1906 return; 1907 } 1908 1909 /* Compute 24 bits worth of difference. */ 1910 diff = qib_cmp24(psn, qp->r_psn); 1911 if (unlikely(diff)) { 1912 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) 1913 return; 1914 goto send_ack; 1915 } 1916 1917 /* Check for opcode sequence errors. */ 1918 switch (qp->r_state) { 1919 case OP(SEND_FIRST): 1920 case OP(SEND_MIDDLE): 1921 if (opcode == OP(SEND_MIDDLE) || 1922 opcode == OP(SEND_LAST) || 1923 opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 1924 break; 1925 goto nack_inv; 1926 1927 case OP(RDMA_WRITE_FIRST): 1928 case OP(RDMA_WRITE_MIDDLE): 1929 if (opcode == OP(RDMA_WRITE_MIDDLE) || 1930 opcode == OP(RDMA_WRITE_LAST) || 1931 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 1932 break; 1933 goto nack_inv; 1934 1935 default: 1936 if (opcode == OP(SEND_MIDDLE) || 1937 opcode == OP(SEND_LAST) || 1938 opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 1939 opcode == OP(RDMA_WRITE_MIDDLE) || 1940 opcode == OP(RDMA_WRITE_LAST) || 1941 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 1942 goto nack_inv; 1943 /* 1944 * Note that it is up to the requester to not send a new 1945 * RDMA read or atomic operation before receiving an ACK 1946 * for the previous operation. 1947 */ 1948 break; 1949 } 1950 1951 if (qp->state == IB_QPS_RTR && !(qp->r_flags & QIB_R_COMM_EST)) { 1952 qp->r_flags |= QIB_R_COMM_EST; 1953 if (qp->ibqp.event_handler) { 1954 struct ib_event ev; 1955 1956 ev.device = qp->ibqp.device; 1957 ev.element.qp = &qp->ibqp; 1958 ev.event = IB_EVENT_COMM_EST; 1959 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 1960 } 1961 } 1962 1963 /* OK, process the packet. */ 1964 switch (opcode) { 1965 case OP(SEND_FIRST): 1966 ret = qib_get_rwqe(qp, 0); 1967 if (ret < 0) 1968 goto nack_op_err; 1969 if (!ret) 1970 goto rnr_nak; 1971 qp->r_rcv_len = 0; 1972 /* FALLTHROUGH */ 1973 case OP(SEND_MIDDLE): 1974 case OP(RDMA_WRITE_MIDDLE): 1975 send_middle: 1976 /* Check for invalid length PMTU or posted rwqe len. */ 1977 if (unlikely(tlen != (hdrsize + pmtu + 4))) 1978 goto nack_inv; 1979 qp->r_rcv_len += pmtu; 1980 if (unlikely(qp->r_rcv_len > qp->r_len)) 1981 goto nack_inv; 1982 qib_copy_sge(&qp->r_sge, data, pmtu, 1); 1983 break; 1984 1985 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 1986 /* consume RWQE */ 1987 ret = qib_get_rwqe(qp, 1); 1988 if (ret < 0) 1989 goto nack_op_err; 1990 if (!ret) 1991 goto rnr_nak; 1992 goto send_last_imm; 1993 1994 case OP(SEND_ONLY): 1995 case OP(SEND_ONLY_WITH_IMMEDIATE): 1996 ret = qib_get_rwqe(qp, 0); 1997 if (ret < 0) 1998 goto nack_op_err; 1999 if (!ret) 2000 goto rnr_nak; 2001 qp->r_rcv_len = 0; 2002 if (opcode == OP(SEND_ONLY)) 2003 goto no_immediate_data; 2004 /* FALLTHROUGH for SEND_ONLY_WITH_IMMEDIATE */ 2005 case OP(SEND_LAST_WITH_IMMEDIATE): 2006 send_last_imm: 2007 wc.ex.imm_data = ohdr->u.imm_data; 2008 hdrsize += 4; 2009 wc.wc_flags = IB_WC_WITH_IMM; 2010 goto send_last; 2011 case OP(SEND_LAST): 2012 case OP(RDMA_WRITE_LAST): 2013 no_immediate_data: 2014 wc.wc_flags = 0; 2015 wc.ex.imm_data = 0; 2016 send_last: 2017 /* Get the number of bytes the message was padded by. */ 2018 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 2019 /* Check for invalid length. */ 2020 /* XXX LAST len should be >= 1 */ 2021 if (unlikely(tlen < (hdrsize + pad + 4))) 2022 goto nack_inv; 2023 /* Don't count the CRC. */ 2024 tlen -= (hdrsize + pad + 4); 2025 wc.byte_len = tlen + qp->r_rcv_len; 2026 if (unlikely(wc.byte_len > qp->r_len)) 2027 goto nack_inv; 2028 qib_copy_sge(&qp->r_sge, data, tlen, 1); 2029 qib_put_ss(&qp->r_sge); 2030 qp->r_msn++; 2031 if (!test_and_clear_bit(QIB_R_WRID_VALID, &qp->r_aflags)) 2032 break; 2033 wc.wr_id = qp->r_wr_id; 2034 wc.status = IB_WC_SUCCESS; 2035 if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) || 2036 opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 2037 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 2038 else 2039 wc.opcode = IB_WC_RECV; 2040 wc.qp = &qp->ibqp; 2041 wc.src_qp = qp->remote_qpn; 2042 wc.slid = qp->remote_ah_attr.dlid; 2043 wc.sl = qp->remote_ah_attr.sl; 2044 /* zero fields that are N/A */ 2045 wc.vendor_err = 0; 2046 wc.pkey_index = 0; 2047 wc.dlid_path_bits = 0; 2048 wc.port_num = 0; 2049 /* Signal completion event if the solicited bit is set. */ 2050 qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 2051 (ohdr->bth[0] & 2052 cpu_to_be32(IB_BTH_SOLICITED)) != 0); 2053 break; 2054 2055 case OP(RDMA_WRITE_FIRST): 2056 case OP(RDMA_WRITE_ONLY): 2057 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 2058 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 2059 goto nack_inv; 2060 /* consume RWQE */ 2061 reth = &ohdr->u.rc.reth; 2062 hdrsize += sizeof(*reth); 2063 qp->r_len = be32_to_cpu(reth->length); 2064 qp->r_rcv_len = 0; 2065 qp->r_sge.sg_list = NULL; 2066 if (qp->r_len != 0) { 2067 u32 rkey = be32_to_cpu(reth->rkey); 2068 u64 vaddr = be64_to_cpu(reth->vaddr); 2069 int ok; 2070 2071 /* Check rkey & NAK */ 2072 ok = qib_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, 2073 rkey, IB_ACCESS_REMOTE_WRITE); 2074 if (unlikely(!ok)) 2075 goto nack_acc; 2076 qp->r_sge.num_sge = 1; 2077 } else { 2078 qp->r_sge.num_sge = 0; 2079 qp->r_sge.sge.mr = NULL; 2080 qp->r_sge.sge.vaddr = NULL; 2081 qp->r_sge.sge.length = 0; 2082 qp->r_sge.sge.sge_length = 0; 2083 } 2084 if (opcode == OP(RDMA_WRITE_FIRST)) 2085 goto send_middle; 2086 else if (opcode == OP(RDMA_WRITE_ONLY)) 2087 goto no_immediate_data; 2088 ret = qib_get_rwqe(qp, 1); 2089 if (ret < 0) 2090 goto nack_op_err; 2091 if (!ret) 2092 goto rnr_nak; 2093 wc.ex.imm_data = ohdr->u.rc.imm_data; 2094 hdrsize += 4; 2095 wc.wc_flags = IB_WC_WITH_IMM; 2096 goto send_last; 2097 2098 case OP(RDMA_READ_REQUEST): { 2099 struct qib_ack_entry *e; 2100 u32 len; 2101 u8 next; 2102 2103 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 2104 goto nack_inv; 2105 next = qp->r_head_ack_queue + 1; 2106 /* s_ack_queue is size QIB_MAX_RDMA_ATOMIC+1 so use > not >= */ 2107 if (next > QIB_MAX_RDMA_ATOMIC) 2108 next = 0; 2109 spin_lock_irqsave(&qp->s_lock, flags); 2110 if (unlikely(next == qp->s_tail_ack_queue)) { 2111 if (!qp->s_ack_queue[next].sent) 2112 goto nack_inv_unlck; 2113 qib_update_ack_queue(qp, next); 2114 } 2115 e = &qp->s_ack_queue[qp->r_head_ack_queue]; 2116 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { 2117 qib_put_mr(e->rdma_sge.mr); 2118 e->rdma_sge.mr = NULL; 2119 } 2120 reth = &ohdr->u.rc.reth; 2121 len = be32_to_cpu(reth->length); 2122 if (len) { 2123 u32 rkey = be32_to_cpu(reth->rkey); 2124 u64 vaddr = be64_to_cpu(reth->vaddr); 2125 int ok; 2126 2127 /* Check rkey & NAK */ 2128 ok = qib_rkey_ok(qp, &e->rdma_sge, len, vaddr, 2129 rkey, IB_ACCESS_REMOTE_READ); 2130 if (unlikely(!ok)) 2131 goto nack_acc_unlck; 2132 /* 2133 * Update the next expected PSN. We add 1 later 2134 * below, so only add the remainder here. 2135 */ 2136 if (len > pmtu) 2137 qp->r_psn += (len - 1) / pmtu; 2138 } else { 2139 e->rdma_sge.mr = NULL; 2140 e->rdma_sge.vaddr = NULL; 2141 e->rdma_sge.length = 0; 2142 e->rdma_sge.sge_length = 0; 2143 } 2144 e->opcode = opcode; 2145 e->sent = 0; 2146 e->psn = psn; 2147 e->lpsn = qp->r_psn; 2148 /* 2149 * We need to increment the MSN here instead of when we 2150 * finish sending the result since a duplicate request would 2151 * increment it more than once. 2152 */ 2153 qp->r_msn++; 2154 qp->r_psn++; 2155 qp->r_state = opcode; 2156 qp->r_nak_state = 0; 2157 qp->r_head_ack_queue = next; 2158 2159 /* Schedule the send tasklet. */ 2160 qp->s_flags |= QIB_S_RESP_PENDING; 2161 qib_schedule_send(qp); 2162 2163 goto sunlock; 2164 } 2165 2166 case OP(COMPARE_SWAP): 2167 case OP(FETCH_ADD): { 2168 struct ib_atomic_eth *ateth; 2169 struct qib_ack_entry *e; 2170 u64 vaddr; 2171 atomic64_t *maddr; 2172 u64 sdata; 2173 u32 rkey; 2174 u8 next; 2175 2176 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) 2177 goto nack_inv; 2178 next = qp->r_head_ack_queue + 1; 2179 if (next > QIB_MAX_RDMA_ATOMIC) 2180 next = 0; 2181 spin_lock_irqsave(&qp->s_lock, flags); 2182 if (unlikely(next == qp->s_tail_ack_queue)) { 2183 if (!qp->s_ack_queue[next].sent) 2184 goto nack_inv_unlck; 2185 qib_update_ack_queue(qp, next); 2186 } 2187 e = &qp->s_ack_queue[qp->r_head_ack_queue]; 2188 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { 2189 qib_put_mr(e->rdma_sge.mr); 2190 e->rdma_sge.mr = NULL; 2191 } 2192 ateth = &ohdr->u.atomic_eth; 2193 vaddr = ((u64) be32_to_cpu(ateth->vaddr[0]) << 32) | 2194 be32_to_cpu(ateth->vaddr[1]); 2195 if (unlikely(vaddr & (sizeof(u64) - 1))) 2196 goto nack_inv_unlck; 2197 rkey = be32_to_cpu(ateth->rkey); 2198 /* Check rkey & NAK */ 2199 if (unlikely(!qib_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), 2200 vaddr, rkey, 2201 IB_ACCESS_REMOTE_ATOMIC))) 2202 goto nack_acc_unlck; 2203 /* Perform atomic OP and save result. */ 2204 maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 2205 sdata = be64_to_cpu(ateth->swap_data); 2206 e->atomic_data = (opcode == OP(FETCH_ADD)) ? 2207 (u64) atomic64_add_return(sdata, maddr) - sdata : 2208 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 2209 be64_to_cpu(ateth->compare_data), 2210 sdata); 2211 qib_put_mr(qp->r_sge.sge.mr); 2212 qp->r_sge.num_sge = 0; 2213 e->opcode = opcode; 2214 e->sent = 0; 2215 e->psn = psn; 2216 e->lpsn = psn; 2217 qp->r_msn++; 2218 qp->r_psn++; 2219 qp->r_state = opcode; 2220 qp->r_nak_state = 0; 2221 qp->r_head_ack_queue = next; 2222 2223 /* Schedule the send tasklet. */ 2224 qp->s_flags |= QIB_S_RESP_PENDING; 2225 qib_schedule_send(qp); 2226 2227 goto sunlock; 2228 } 2229 2230 default: 2231 /* NAK unknown opcodes. */ 2232 goto nack_inv; 2233 } 2234 qp->r_psn++; 2235 qp->r_state = opcode; 2236 qp->r_ack_psn = psn; 2237 qp->r_nak_state = 0; 2238 /* Send an ACK if requested or required. */ 2239 if (psn & (1 << 31)) 2240 goto send_ack; 2241 return; 2242 2243 rnr_nak: 2244 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; 2245 qp->r_ack_psn = qp->r_psn; 2246 /* Queue RNR NAK for later */ 2247 if (list_empty(&qp->rspwait)) { 2248 qp->r_flags |= QIB_R_RSP_NAK; 2249 atomic_inc(&qp->refcount); 2250 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2251 } 2252 return; 2253 2254 nack_op_err: 2255 qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 2256 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; 2257 qp->r_ack_psn = qp->r_psn; 2258 /* Queue NAK for later */ 2259 if (list_empty(&qp->rspwait)) { 2260 qp->r_flags |= QIB_R_RSP_NAK; 2261 atomic_inc(&qp->refcount); 2262 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2263 } 2264 return; 2265 2266 nack_inv_unlck: 2267 spin_unlock_irqrestore(&qp->s_lock, flags); 2268 nack_inv: 2269 qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 2270 qp->r_nak_state = IB_NAK_INVALID_REQUEST; 2271 qp->r_ack_psn = qp->r_psn; 2272 /* Queue NAK for later */ 2273 if (list_empty(&qp->rspwait)) { 2274 qp->r_flags |= QIB_R_RSP_NAK; 2275 atomic_inc(&qp->refcount); 2276 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2277 } 2278 return; 2279 2280 nack_acc_unlck: 2281 spin_unlock_irqrestore(&qp->s_lock, flags); 2282 nack_acc: 2283 qib_rc_error(qp, IB_WC_LOC_PROT_ERR); 2284 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; 2285 qp->r_ack_psn = qp->r_psn; 2286 send_ack: 2287 qib_send_rc_ack(qp); 2288 return; 2289 2290 sunlock: 2291 spin_unlock_irqrestore(&qp->s_lock, flags); 2292 } 2293