1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved. 4 */ 5 6 /* 7 * The rdma_rxe driver supports type 1 or type 2B memory windows. 8 * Type 1 MWs are created by ibv_alloc_mw() verbs calls and bound by 9 * ibv_bind_mw() calls. Type 2 MWs are also created by ibv_alloc_mw() 10 * but bound by bind_mw work requests. The ibv_bind_mw() call is converted 11 * by libibverbs to a bind_mw work request. 12 */ 13 14 #include "rxe.h" 15 16 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 17 { 18 struct rxe_mw *mw = to_rmw(ibmw); 19 struct rxe_pd *pd = to_rpd(ibmw->pd); 20 struct rxe_dev *rxe = to_rdev(ibmw->device); 21 int ret; 22 23 rxe_get(pd); 24 25 ret = rxe_add_to_pool(&rxe->mw_pool, mw); 26 if (ret) { 27 rxe_put(pd); 28 return ret; 29 } 30 31 mw->rkey = ibmw->rkey = (mw->elem.index << 8) | rxe_get_next_key(-1); 32 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ? 33 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID; 34 spin_lock_init(&mw->lock); 35 36 return 0; 37 } 38 39 int rxe_dealloc_mw(struct ib_mw *ibmw) 40 { 41 struct rxe_mw *mw = to_rmw(ibmw); 42 43 rxe_put(mw); 44 45 return 0; 46 } 47 48 static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 49 struct rxe_mw *mw, struct rxe_mr *mr) 50 { 51 u32 key = wqe->wr.wr.mw.rkey & 0xff; 52 53 if (mw->ibmw.type == IB_MW_TYPE_1) { 54 if (unlikely(mw->state != RXE_MW_STATE_VALID)) { 55 pr_err_once( 56 "attempt to bind a type 1 MW not in the valid state\n"); 57 return -EINVAL; 58 } 59 60 /* o10-36.2.2 */ 61 if (unlikely((mw->access & IB_ZERO_BASED))) { 62 pr_err_once("attempt to bind a zero based type 1 MW\n"); 63 return -EINVAL; 64 } 65 } 66 67 if (mw->ibmw.type == IB_MW_TYPE_2) { 68 /* o10-37.2.30 */ 69 if (unlikely(mw->state != RXE_MW_STATE_FREE)) { 70 pr_err_once( 71 "attempt to bind a type 2 MW not in the free state\n"); 72 return -EINVAL; 73 } 74 75 /* C10-72 */ 76 if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) { 77 pr_err_once( 78 "attempt to bind type 2 MW with qp with different PD\n"); 79 return -EINVAL; 80 } 81 82 /* o10-37.2.40 */ 83 if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) { 84 pr_err_once( 85 "attempt to invalidate type 2 MW by binding with NULL or zero length MR\n"); 86 return -EINVAL; 87 } 88 } 89 90 if (unlikely(key == (mw->rkey & 0xff))) { 91 pr_err_once("attempt to bind MW with same key\n"); 92 return -EINVAL; 93 } 94 95 /* remaining checks only apply to a nonzero MR */ 96 if (!mr) 97 return 0; 98 99 if (unlikely(mr->access & IB_ZERO_BASED)) { 100 pr_err_once("attempt to bind MW to zero based MR\n"); 101 return -EINVAL; 102 } 103 104 /* C10-73 */ 105 if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) { 106 pr_err_once( 107 "attempt to bind an MW to an MR without bind access\n"); 108 return -EINVAL; 109 } 110 111 /* C10-74 */ 112 if (unlikely((mw->access & 113 (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC)) && 114 !(mr->access & IB_ACCESS_LOCAL_WRITE))) { 115 pr_err_once( 116 "attempt to bind an writeable MW to an MR without local write access\n"); 117 return -EINVAL; 118 } 119 120 /* C10-75 */ 121 if (mw->access & IB_ZERO_BASED) { 122 if (unlikely(wqe->wr.wr.mw.length > mr->cur_map_set->length)) { 123 pr_err_once( 124 "attempt to bind a ZB MW outside of the MR\n"); 125 return -EINVAL; 126 } 127 } else { 128 if (unlikely((wqe->wr.wr.mw.addr < mr->cur_map_set->iova) || 129 ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) > 130 (mr->cur_map_set->iova + mr->cur_map_set->length)))) { 131 pr_err_once( 132 "attempt to bind a VA MW outside of the MR\n"); 133 return -EINVAL; 134 } 135 } 136 137 return 0; 138 } 139 140 static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, 141 struct rxe_mw *mw, struct rxe_mr *mr) 142 { 143 u32 key = wqe->wr.wr.mw.rkey & 0xff; 144 145 mw->rkey = (mw->rkey & ~0xff) | key; 146 mw->access = wqe->wr.wr.mw.access; 147 mw->state = RXE_MW_STATE_VALID; 148 mw->addr = wqe->wr.wr.mw.addr; 149 mw->length = wqe->wr.wr.mw.length; 150 151 if (mw->mr) { 152 rxe_put(mw->mr); 153 atomic_dec(&mw->mr->num_mw); 154 mw->mr = NULL; 155 } 156 157 if (mw->length) { 158 mw->mr = mr; 159 atomic_inc(&mr->num_mw); 160 rxe_get(mr); 161 } 162 163 if (mw->ibmw.type == IB_MW_TYPE_2) { 164 rxe_get(qp); 165 mw->qp = qp; 166 } 167 } 168 169 int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) 170 { 171 int ret; 172 struct rxe_mw *mw; 173 struct rxe_mr *mr; 174 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 175 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey; 176 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey; 177 178 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8); 179 if (unlikely(!mw)) { 180 ret = -EINVAL; 181 goto err; 182 } 183 184 if (unlikely(mw->rkey != mw_rkey)) { 185 ret = -EINVAL; 186 goto err_drop_mw; 187 } 188 189 if (likely(wqe->wr.wr.mw.length)) { 190 mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8); 191 if (unlikely(!mr)) { 192 ret = -EINVAL; 193 goto err_drop_mw; 194 } 195 196 if (unlikely(mr->lkey != mr_lkey)) { 197 ret = -EINVAL; 198 goto err_drop_mr; 199 } 200 } else { 201 mr = NULL; 202 } 203 204 spin_lock_bh(&mw->lock); 205 206 ret = rxe_check_bind_mw(qp, wqe, mw, mr); 207 if (ret) 208 goto err_unlock; 209 210 rxe_do_bind_mw(qp, wqe, mw, mr); 211 err_unlock: 212 spin_unlock_bh(&mw->lock); 213 err_drop_mr: 214 if (mr) 215 rxe_put(mr); 216 err_drop_mw: 217 rxe_put(mw); 218 err: 219 return ret; 220 } 221 222 static int rxe_check_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw) 223 { 224 if (unlikely(mw->state == RXE_MW_STATE_INVALID)) 225 return -EINVAL; 226 227 /* o10-37.2.26 */ 228 if (unlikely(mw->ibmw.type == IB_MW_TYPE_1)) 229 return -EINVAL; 230 231 return 0; 232 } 233 234 static void rxe_do_invalidate_mw(struct rxe_mw *mw) 235 { 236 struct rxe_qp *qp; 237 struct rxe_mr *mr; 238 239 /* valid type 2 MW will always have a QP pointer */ 240 qp = mw->qp; 241 mw->qp = NULL; 242 rxe_put(qp); 243 244 /* valid type 2 MW will always have an MR pointer */ 245 mr = mw->mr; 246 mw->mr = NULL; 247 atomic_dec(&mr->num_mw); 248 rxe_put(mr); 249 250 mw->access = 0; 251 mw->addr = 0; 252 mw->length = 0; 253 mw->state = RXE_MW_STATE_FREE; 254 } 255 256 int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey) 257 { 258 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 259 struct rxe_mw *mw; 260 int ret; 261 262 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8); 263 if (!mw) { 264 ret = -EINVAL; 265 goto err; 266 } 267 268 if (rkey != mw->rkey) { 269 ret = -EINVAL; 270 goto err_drop_ref; 271 } 272 273 spin_lock_bh(&mw->lock); 274 275 ret = rxe_check_invalidate_mw(qp, mw); 276 if (ret) 277 goto err_unlock; 278 279 rxe_do_invalidate_mw(mw); 280 err_unlock: 281 spin_unlock_bh(&mw->lock); 282 err_drop_ref: 283 rxe_put(mw); 284 err: 285 return ret; 286 } 287 288 struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey) 289 { 290 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); 291 struct rxe_pd *pd = to_rpd(qp->ibqp.pd); 292 struct rxe_mw *mw; 293 int index = rkey >> 8; 294 295 mw = rxe_pool_get_index(&rxe->mw_pool, index); 296 if (!mw) 297 return NULL; 298 299 if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd || 300 (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) || 301 (mw->length == 0) || 302 (access && !(access & mw->access)) || 303 mw->state != RXE_MW_STATE_VALID)) { 304 rxe_put(mw); 305 return NULL; 306 } 307 308 return mw; 309 } 310 311 void rxe_mw_cleanup(struct rxe_pool_elem *elem) 312 { 313 struct rxe_mw *mw = container_of(elem, typeof(*mw), elem); 314 struct rxe_pd *pd = to_rpd(mw->ibmw.pd); 315 316 rxe_put(pd); 317 318 if (mw->mr) { 319 struct rxe_mr *mr = mw->mr; 320 321 mw->mr = NULL; 322 atomic_dec(&mr->num_mw); 323 rxe_put(mr); 324 } 325 326 if (mw->qp) { 327 struct rxe_qp *qp = mw->qp; 328 329 mw->qp = NULL; 330 rxe_put(qp); 331 } 332 333 mw->access = 0; 334 mw->addr = 0; 335 mw->length = 0; 336 mw->state = RXE_MW_STATE_INVALID; 337 } 338