1 /* 2 * Copyright(c) 2016 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <rdma/rdmavt_qp.h> 49 #include <rdma/ib_hdrs.h> 50 51 /* 52 * Convert the AETH credit code into the number of credits. 53 */ 54 static const u16 credit_table[31] = { 55 0, /* 0 */ 56 1, /* 1 */ 57 2, /* 2 */ 58 3, /* 3 */ 59 4, /* 4 */ 60 6, /* 5 */ 61 8, /* 6 */ 62 12, /* 7 */ 63 16, /* 8 */ 64 24, /* 9 */ 65 32, /* A */ 66 48, /* B */ 67 64, /* C */ 68 96, /* D */ 69 128, /* E */ 70 192, /* F */ 71 256, /* 10 */ 72 384, /* 11 */ 73 512, /* 12 */ 74 768, /* 13 */ 75 1024, /* 14 */ 76 1536, /* 15 */ 77 2048, /* 16 */ 78 3072, /* 17 */ 79 4096, /* 18 */ 80 6144, /* 19 */ 81 8192, /* 1A */ 82 12288, /* 1B */ 83 16384, /* 1C */ 84 24576, /* 1D */ 85 32768 /* 1E */ 86 }; 87 88 /** 89 * rvt_compute_aeth - compute the AETH (syndrome + MSN) 90 * @qp: the queue pair to compute the AETH for 91 * 92 * Returns the AETH. 93 */ 94 __be32 rvt_compute_aeth(struct rvt_qp *qp) 95 { 96 u32 aeth = qp->r_msn & IB_MSN_MASK; 97 98 if (qp->ibqp.srq) { 99 /* 100 * Shared receive queues don't generate credits. 101 * Set the credit field to the invalid value. 102 */ 103 aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT; 104 } else { 105 u32 min, max, x; 106 u32 credits; 107 u32 head; 108 u32 tail; 109 110 credits = READ_ONCE(qp->r_rq.kwq->count); 111 if (credits == 0) { 112 /* sanity check pointers before trusting them */ 113 if (qp->ip) { 114 head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head); 115 tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail); 116 } else { 117 head = READ_ONCE(qp->r_rq.kwq->head); 118 tail = READ_ONCE(qp->r_rq.kwq->tail); 119 } 120 if (head >= qp->r_rq.size) 121 head = 0; 122 if (tail >= qp->r_rq.size) 123 tail = 0; 124 /* 125 * Compute the number of credits available (RWQEs). 126 * There is a small chance that the pair of reads are 127 * not atomic, which is OK, since the fuzziness is 128 * resolved as further ACKs go out. 129 */ 130 credits = rvt_get_rq_count(&qp->r_rq, head, tail); 131 } 132 /* 133 * Binary search the credit table to find the code to 134 * use. 135 */ 136 min = 0; 137 max = 31; 138 for (;;) { 139 x = (min + max) / 2; 140 if (credit_table[x] == credits) 141 break; 142 if (credit_table[x] > credits) { 143 max = x; 144 } else { 145 if (min == x) 146 break; 147 min = x; 148 } 149 } 150 aeth |= x << IB_AETH_CREDIT_SHIFT; 151 } 152 return cpu_to_be32(aeth); 153 } 154 EXPORT_SYMBOL(rvt_compute_aeth); 155 156 /** 157 * rvt_get_credit - flush the send work queue of a QP 158 * @qp: the qp who's send work queue to flush 159 * @aeth: the Acknowledge Extended Transport Header 160 * 161 * The QP s_lock should be held. 162 */ 163 void rvt_get_credit(struct rvt_qp *qp, u32 aeth) 164 { 165 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); 166 u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK; 167 168 lockdep_assert_held(&qp->s_lock); 169 /* 170 * If the credit is invalid, we can send 171 * as many packets as we like. Otherwise, we have to 172 * honor the credit field. 173 */ 174 if (credit == IB_AETH_CREDIT_INVAL) { 175 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { 176 qp->s_flags |= RVT_S_UNLIMITED_CREDIT; 177 if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { 178 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; 179 rdi->driver_f.schedule_send(qp); 180 } 181 } 182 } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { 183 /* Compute new LSN (i.e., MSN + credit) */ 184 credit = (aeth + credit_table[credit]) & IB_MSN_MASK; 185 if (rvt_cmp_msn(credit, qp->s_lsn) > 0) { 186 qp->s_lsn = credit; 187 if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { 188 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; 189 rdi->driver_f.schedule_send(qp); 190 } 191 } 192 } 193 } 194 EXPORT_SYMBOL(rvt_get_credit); 195 196 /** 197 * rvt_restart_sge - rewind the sge state for a wqe 198 * @ss: the sge state pointer 199 * @wqe: the wqe to rewind 200 * @len: the data length from the start of the wqe in bytes 201 * 202 * Returns the remaining data length. 203 */ 204 u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len) 205 { 206 ss->sge = wqe->sg_list[0]; 207 ss->sg_list = wqe->sg_list + 1; 208 ss->num_sge = wqe->wr.num_sge; 209 ss->total_len = wqe->length; 210 rvt_skip_sge(ss, len, false); 211 return wqe->length - len; 212 } 213 EXPORT_SYMBOL(rvt_restart_sge); 214 215