1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved. 4 */ 5 6 #include "rxe.h" 7 8 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 9 { 10 struct rxe_mw *mw = to_rmw(ibmw); 11 struct rxe_pd *pd = to_rpd(ibmw->pd); 12 struct rxe_dev *rxe = to_rdev(ibmw->device); 13 int ret; 14 15 rxe_add_ref(pd); 16 17 ret = rxe_add_to_pool(&rxe->mw_pool, mw); 18 if (ret) { 19 rxe_drop_ref(pd); 20 return ret; 21 } 22 23 rxe_add_index(mw); 24 mw->rkey = ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1); 25 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ? 26 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID; 27 spin_lock_init(&mw->lock); 28 29 return 0; 30 } 31 32 static void rxe_do_dealloc_mw(struct rxe_mw *mw) 33 { 34 if (mw->mr) { 35 struct rxe_mr *mr = mw->mr; 36 37 mw->mr = NULL; 38 atomic_dec(&mr->num_mw); 39 rxe_drop_ref(mr); 40 } 41 42 if (mw->qp) { 43 struct rxe_qp *qp = mw->qp; 44 45 mw->qp = NULL; 46 rxe_drop_ref(qp); 47 } 48 49 mw->access = 0; 50 mw->addr = 0; 51 mw->length = 0; 52 mw->state = RXE_MW_STATE_INVALID; 53 } 54 55 int rxe_dealloc_mw(struct ib_mw *ibmw) 56 { 57 struct rxe_mw *mw = to_rmw(ibmw); 58 struct rxe_pd *pd = to_rpd(ibmw->pd); 59 unsigned long flags; 60 61 spin_lock_irqsave(&mw->lock, flags); 62 rxe_do_dealloc_mw(mw); 63 spin_unlock_irqrestore(&mw->lock, flags); 64 65 rxe_drop_ref(mw); 66 rxe_drop_ref(pd); 67 68 return 0; 69 } 70 71 static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 72 struct rxe_mw *mw, struct rxe_mr *mr) 73 { 74 u32 key = wqe->wr.wr.mw.rkey & 0xff; 75 76 if (mw->ibmw.type == IB_MW_TYPE_1) { 77 if (unlikely(mw->state != RXE_MW_STATE_VALID)) { 78 pr_err_once( 79 "attempt to bind a type 1 MW not in the valid state\n"); 80 return -EINVAL; 81 } 82 83 /* o10-36.2.2 */ 84 if (unlikely((mw->access & IB_ZERO_BASED))) { 85 pr_err_once("attempt to bind a zero based type 1 MW\n"); 86 return -EINVAL; 87 } 88 } 89 90 if (mw->ibmw.type == IB_MW_TYPE_2) { 91 /* o10-37.2.30 */ 92 if (unlikely(mw->state != RXE_MW_STATE_FREE)) { 93 pr_err_once( 94 "attempt to bind a type 2 MW not in the free state\n"); 95 return -EINVAL; 96 } 97 98 /* C10-72 */ 99 if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) { 100 pr_err_once( 101 "attempt to bind type 2 MW with qp with different PD\n"); 102 return -EINVAL; 103 } 104 105 /* o10-37.2.40 */ 106 if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) { 107 pr_err_once( 108 "attempt to invalidate type 2 MW by binding with NULL or zero length MR\n"); 109 return -EINVAL; 110 } 111 } 112 113 if (unlikely(key == (mw->rkey & 0xff))) { 114 pr_err_once("attempt to bind MW with same key\n"); 115 return -EINVAL; 116 } 117 118 /* remaining checks only apply to a nonzero MR */ 119 if (!mr) 120 return 0; 121 122 if (unlikely(mr->access & IB_ZERO_BASED)) { 123 pr_err_once("attempt to bind MW to zero based MR\n"); 124 return -EINVAL; 125 } 126 127 /* C10-73 */ 128 if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) { 129 pr_err_once( 130 "attempt to bind an MW to an MR without bind access\n"); 131 return -EINVAL; 132 } 133 134 /* C10-74 */ 135 if (unlikely((mw->access & 136 (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC)) && 137 !(mr->access & IB_ACCESS_LOCAL_WRITE))) { 138 pr_err_once( 139 "attempt to bind an writeable MW to an MR without local write access\n"); 140 return -EINVAL; 141 } 142 143 /* C10-75 */ 144 if (mw->access & IB_ZERO_BASED) { 145 if (unlikely(wqe->wr.wr.mw.length > mr->cur_map_set->length)) { 146 pr_err_once( 147 "attempt to bind a ZB MW outside of the MR\n"); 148 return -EINVAL; 149 } 150 } else { 151 if (unlikely((wqe->wr.wr.mw.addr < mr->cur_map_set->iova) || 152 ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) > 153 (mr->cur_map_set->iova + mr->cur_map_set->length)))) { 154 pr_err_once( 155 "attempt to bind a VA MW outside of the MR\n"); 156 return -EINVAL; 157 } 158 } 159 160 return 0; 161 } 162 163 static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 164 struct rxe_mw *mw, struct rxe_mr *mr) 165 { 166 u32 key = wqe->wr.wr.mw.rkey & 0xff; 167 168 mw->rkey = (mw->rkey & ~0xff) | key; 169 mw->access = wqe->wr.wr.mw.access; 170 mw->state = RXE_MW_STATE_VALID; 171 mw->addr = wqe->wr.wr.mw.addr; 172 mw->length = wqe->wr.wr.mw.length; 173 174 if (mw->mr) { 175 rxe_drop_ref(mw->mr); 176 atomic_dec(&mw->mr->num_mw); 177 mw->mr = NULL; 178 } 179 180 if (mw->length) { 181 mw->mr = mr; 182 atomic_inc(&mr->num_mw); 183 rxe_add_ref(mr); 184 } 185 186 if (mw->ibmw.type == IB_MW_TYPE_2) { 187 rxe_add_ref(qp); 188 mw->qp = qp; 189 } 190 } 191 192 int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) 193 { 194 int ret; 195 struct rxe_mw *mw; 196 struct rxe_mr *mr; 197 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 198 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey; 199 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey; 200 unsigned long flags; 201 202 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8); 203 if (unlikely(!mw)) { 204 ret = -EINVAL; 205 goto err; 206 } 207 208 if (unlikely(mw->rkey != mw_rkey)) { 209 ret = -EINVAL; 210 goto err_drop_mw; 211 } 212 213 if (likely(wqe->wr.wr.mw.length)) { 214 mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8); 215 if (unlikely(!mr)) { 216 ret = -EINVAL; 217 goto err_drop_mw; 218 } 219 220 if (unlikely(mr->lkey != mr_lkey)) { 221 ret = -EINVAL; 222 goto err_drop_mr; 223 } 224 } else { 225 mr = NULL; 226 } 227 228 spin_lock_irqsave(&mw->lock, flags); 229 230 ret = rxe_check_bind_mw(qp, wqe, mw, mr); 231 if (ret) 232 goto err_unlock; 233 234 rxe_do_bind_mw(qp, wqe, mw, mr); 235 err_unlock: 236 spin_unlock_irqrestore(&mw->lock, flags); 237 err_drop_mr: 238 if (mr) 239 rxe_drop_ref(mr); 240 err_drop_mw: 241 rxe_drop_ref(mw); 242 err: 243 return ret; 244 } 245 246 static int rxe_check_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw) 247 { 248 if (unlikely(mw->state == RXE_MW_STATE_INVALID)) 249 return -EINVAL; 250 251 /* o10-37.2.26 */ 252 if (unlikely(mw->ibmw.type == IB_MW_TYPE_1)) 253 return -EINVAL; 254 255 return 0; 256 } 257 258 static void rxe_do_invalidate_mw(struct rxe_mw *mw) 259 { 260 struct rxe_qp *qp; 261 struct rxe_mr *mr; 262 263 /* valid type 2 MW will always have a QP pointer */ 264 qp = mw->qp; 265 mw->qp = NULL; 266 rxe_drop_ref(qp); 267 268 /* valid type 2 MW will always have an MR pointer */ 269 mr = mw->mr; 270 mw->mr = NULL; 271 atomic_dec(&mr->num_mw); 272 rxe_drop_ref(mr); 273 274 mw->access = 0; 275 mw->addr = 0; 276 mw->length = 0; 277 mw->state = RXE_MW_STATE_FREE; 278 } 279 280 int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) 281 { 282 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 283 unsigned long flags; 284 struct rxe_mw *mw; 285 int ret; 286 287 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8); 288 if (!mw) { 289 ret = -EINVAL; 290 goto err; 291 } 292 293 if (rkey != mw->rkey) { 294 ret = -EINVAL; 295 goto err_drop_ref; 296 } 297 298 spin_lock_irqsave(&mw->lock, flags); 299 300 ret = rxe_check_invalidate_mw(qp, mw); 301 if (ret) 302 goto err_unlock; 303 304 rxe_do_invalidate_mw(mw); 305 err_unlock: 306 spin_unlock_irqrestore(&mw->lock, flags); 307 err_drop_ref: 308 rxe_drop_ref(mw); 309 err: 310 return ret; 311 } 312 313 struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey) 314 { 315 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 316 struct rxe_pd *pd = to_rpd(qp->ibqp.pd); 317 struct rxe_mw *mw; 318 int index = rkey >> 8; 319 320 mw = rxe_pool_get_index(&rxe->mw_pool, index); 321 if (!mw) 322 return NULL; 323 324 if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd || 325 (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) || 326 (mw->length == 0) || 327 (access && !(access & mw->access)) || 328 mw->state != RXE_MW_STATE_VALID)) { 329 rxe_drop_ref(mw); 330 return NULL; 331 } 332 333 return mw; 334 } 335 336 void rxe_mw_cleanup(struct rxe_pool_entry *elem) 337 { 338 struct rxe_mw *mw = container_of(elem, typeof(*mw), pelem); 339 340 rxe_drop_index(mw); 341 } 342