1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018 Hisilicon Limited. 4 */ 5 6 #include <rdma/ib_umem.h> 7 #include <rdma/hns-abi.h> 8 #include "hns_roce_device.h" 9 #include "hns_roce_cmd.h" 10 #include "hns_roce_hem.h" 11 12 void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type) 13 { 14 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 15 struct hns_roce_srq *srq; 16 17 xa_lock(&srq_table->xa); 18 srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1)); 19 if (srq) 20 atomic_inc(&srq->refcount); 21 xa_unlock(&srq_table->xa); 22 23 if (!srq) { 24 dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn); 25 return; 26 } 27 28 srq->event(srq, event_type); 29 30 if (atomic_dec_and_test(&srq->refcount)) 31 complete(&srq->free); 32 } 33 EXPORT_SYMBOL_GPL(hns_roce_srq_event); 34 35 static void hns_roce_ib_srq_event(struct hns_roce_srq *srq, 36 enum hns_roce_event event_type) 37 { 38 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); 39 struct ib_srq *ibsrq = &srq->ibsrq; 40 struct ib_event event; 41 42 if (ibsrq->event_handler) { 43 event.device = ibsrq->device; 44 event.element.srq = ibsrq; 45 switch (event_type) { 46 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 47 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 48 break; 49 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 50 event.event = IB_EVENT_SRQ_ERR; 51 break; 52 default: 53 dev_err(hr_dev->dev, 54 "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n", 55 event_type, srq->srqn); 56 return; 57 } 58 59 ibsrq->event_handler(&event, ibsrq->srq_context); 60 } 61 } 62 63 static int hns_roce_sw2hw_srq(struct hns_roce_dev *dev, 64 struct hns_roce_cmd_mailbox *mailbox, 65 unsigned long srq_num) 66 { 67 return hns_roce_cmd_mbox(dev, mailbox->dma, 0, srq_num, 0, 68 HNS_ROCE_CMD_SW2HW_SRQ, 69 HNS_ROCE_CMD_TIMEOUT_MSECS); 70 } 71 72 static int hns_roce_hw2sw_srq(struct hns_roce_dev *dev, 73 struct hns_roce_cmd_mailbox *mailbox, 74 unsigned long srq_num) 75 { 76 return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, srq_num, 77 mailbox ? 0 : 1, HNS_ROCE_CMD_HW2SW_SRQ, 78 HNS_ROCE_CMD_TIMEOUT_MSECS); 79 } 80 81 static int hns_roce_srq_alloc(struct hns_roce_dev *hr_dev, u32 pdn, u32 cqn, 82 u16 xrcd, struct hns_roce_mtt *hr_mtt, 83 u64 db_rec_addr, struct hns_roce_srq *srq) 84 { 85 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 86 struct hns_roce_cmd_mailbox *mailbox; 87 dma_addr_t dma_handle_wqe; 88 dma_addr_t dma_handle_idx; 89 u64 *mtts_wqe; 90 u64 *mtts_idx; 91 int ret; 92 93 /* Get the physical address of srq buf */ 94 mtts_wqe = hns_roce_table_find(hr_dev, 95 &hr_dev->mr_table.mtt_srqwqe_table, 96 srq->mtt.first_seg, 97 &dma_handle_wqe); 98 if (!mtts_wqe) { 99 dev_err(hr_dev->dev, 100 "SRQ alloc.Failed to find srq buf addr.\n"); 101 return -EINVAL; 102 } 103 104 /* Get physical address of idx que buf */ 105 mtts_idx = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_idx_table, 106 srq->idx_que.mtt.first_seg, 107 &dma_handle_idx); 108 if (!mtts_idx) { 109 dev_err(hr_dev->dev, 110 "SRQ alloc.Failed to find idx que buf addr.\n"); 111 return -EINVAL; 112 } 113 114 ret = hns_roce_bitmap_alloc(&srq_table->bitmap, &srq->srqn); 115 if (ret == -1) { 116 dev_err(hr_dev->dev, "SRQ alloc.Failed to alloc index.\n"); 117 return -ENOMEM; 118 } 119 120 ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn); 121 if (ret) 122 goto err_out; 123 124 ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL)); 125 if (ret) 126 goto err_put; 127 128 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 129 if (IS_ERR(mailbox)) { 130 ret = PTR_ERR(mailbox); 131 goto err_xa; 132 } 133 134 hr_dev->hw->write_srqc(hr_dev, srq, pdn, xrcd, cqn, mailbox->buf, 135 mtts_wqe, mtts_idx, dma_handle_wqe, 136 dma_handle_idx); 137 138 ret = hns_roce_sw2hw_srq(hr_dev, mailbox, srq->srqn); 139 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 140 if (ret) 141 goto err_xa; 142 143 atomic_set(&srq->refcount, 1); 144 init_completion(&srq->free); 145 return ret; 146 147 err_xa: 148 xa_erase(&srq_table->xa, srq->srqn); 149 150 err_put: 151 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); 152 153 err_out: 154 hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR); 155 return ret; 156 } 157 158 static void hns_roce_srq_free(struct hns_roce_dev *hr_dev, 159 struct hns_roce_srq *srq) 160 { 161 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 162 int ret; 163 164 ret = hns_roce_hw2sw_srq(hr_dev, NULL, srq->srqn); 165 if (ret) 166 dev_err(hr_dev->dev, "HW2SW_SRQ failed (%d) for CQN %06lx\n", 167 ret, srq->srqn); 168 169 xa_erase(&srq_table->xa, srq->srqn); 170 171 if (atomic_dec_and_test(&srq->refcount)) 172 complete(&srq->free); 173 wait_for_completion(&srq->free); 174 175 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); 176 hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR); 177 } 178 179 static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq, 180 u32 page_shift) 181 { 182 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 183 struct hns_roce_idx_que *idx_que = &srq->idx_que; 184 u32 bitmap_num; 185 int i; 186 187 bitmap_num = HNS_ROCE_ALOGN_UP(srq->max, 8 * sizeof(u64)); 188 189 idx_que->bitmap = kcalloc(1, bitmap_num / 8, GFP_KERNEL); 190 if (!idx_que->bitmap) 191 return -ENOMEM; 192 193 bitmap_num = bitmap_num / (8 * sizeof(u64)); 194 195 idx_que->buf_size = srq->idx_que.buf_size; 196 197 if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2, 198 &idx_que->idx_buf, page_shift)) { 199 kfree(idx_que->bitmap); 200 return -ENOMEM; 201 } 202 203 for (i = 0; i < bitmap_num; i++) 204 idx_que->bitmap[i] = ~(0UL); 205 206 return 0; 207 } 208 209 int hns_roce_create_srq(struct ib_srq *ib_srq, 210 struct ib_srq_init_attr *srq_init_attr, 211 struct ib_udata *udata) 212 { 213 struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device); 214 struct hns_roce_ib_create_srq_resp resp = {}; 215 struct hns_roce_srq *srq = to_hr_srq(ib_srq); 216 int srq_desc_size; 217 int srq_buf_size; 218 u32 page_shift; 219 int ret = 0; 220 u32 npages; 221 u32 cqn; 222 223 /* Check the actual SRQ wqe and SRQ sge num */ 224 if (srq_init_attr->attr.max_wr >= hr_dev->caps.max_srq_wrs || 225 srq_init_attr->attr.max_sge > hr_dev->caps.max_srq_sges) 226 return -EINVAL; 227 228 mutex_init(&srq->mutex); 229 spin_lock_init(&srq->lock); 230 231 srq->max = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1); 232 srq->max_gs = srq_init_attr->attr.max_sge; 233 234 srq_desc_size = max(16, 16 * srq->max_gs); 235 236 srq->wqe_shift = ilog2(srq_desc_size); 237 238 srq_buf_size = srq->max * srq_desc_size; 239 240 srq->idx_que.entry_sz = HNS_ROCE_IDX_QUE_ENTRY_SZ; 241 srq->idx_que.buf_size = srq->max * srq->idx_que.entry_sz; 242 srq->mtt.mtt_type = MTT_TYPE_SRQWQE; 243 srq->idx_que.mtt.mtt_type = MTT_TYPE_IDX; 244 245 if (udata) { 246 struct hns_roce_ib_create_srq ucmd; 247 248 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) 249 return -EFAULT; 250 251 srq->umem = 252 ib_umem_get(udata, ucmd.buf_addr, srq_buf_size, 0, 0); 253 if (IS_ERR(srq->umem)) 254 return PTR_ERR(srq->umem); 255 256 if (hr_dev->caps.srqwqe_buf_pg_sz) { 257 npages = (ib_umem_page_count(srq->umem) + 258 (1 << hr_dev->caps.srqwqe_buf_pg_sz) - 1) / 259 (1 << hr_dev->caps.srqwqe_buf_pg_sz); 260 page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz; 261 ret = hns_roce_mtt_init(hr_dev, npages, 262 page_shift, 263 &srq->mtt); 264 } else 265 ret = hns_roce_mtt_init(hr_dev, 266 ib_umem_page_count(srq->umem), 267 srq->umem->page_shift, 268 &srq->mtt); 269 if (ret) 270 goto err_buf; 271 272 ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->mtt, srq->umem); 273 if (ret) 274 goto err_srq_mtt; 275 276 /* config index queue BA */ 277 srq->idx_que.umem = ib_umem_get(udata, ucmd.que_addr, 278 srq->idx_que.buf_size, 0, 0); 279 if (IS_ERR(srq->idx_que.umem)) { 280 dev_err(hr_dev->dev, 281 "ib_umem_get error for index queue\n"); 282 ret = PTR_ERR(srq->idx_que.umem); 283 goto err_srq_mtt; 284 } 285 286 if (hr_dev->caps.idx_buf_pg_sz) { 287 npages = (ib_umem_page_count(srq->idx_que.umem) + 288 (1 << hr_dev->caps.idx_buf_pg_sz) - 1) / 289 (1 << hr_dev->caps.idx_buf_pg_sz); 290 page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz; 291 ret = hns_roce_mtt_init(hr_dev, npages, 292 page_shift, &srq->idx_que.mtt); 293 } else { 294 ret = hns_roce_mtt_init(hr_dev, 295 ib_umem_page_count(srq->idx_que.umem), 296 srq->idx_que.umem->page_shift, 297 &srq->idx_que.mtt); 298 } 299 300 if (ret) { 301 dev_err(hr_dev->dev, 302 "hns_roce_mtt_init error for idx que\n"); 303 goto err_idx_mtt; 304 } 305 306 ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->idx_que.mtt, 307 srq->idx_que.umem); 308 if (ret) { 309 dev_err(hr_dev->dev, 310 "hns_roce_ib_umem_write_mtt error for idx que\n"); 311 goto err_idx_buf; 312 } 313 } else { 314 page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz; 315 if (hns_roce_buf_alloc(hr_dev, srq_buf_size, 316 (1 << page_shift) * 2, &srq->buf, 317 page_shift)) 318 return -ENOMEM; 319 320 srq->head = 0; 321 srq->tail = srq->max - 1; 322 323 ret = hns_roce_mtt_init(hr_dev, srq->buf.npages, 324 srq->buf.page_shift, &srq->mtt); 325 if (ret) 326 goto err_buf; 327 328 ret = hns_roce_buf_write_mtt(hr_dev, &srq->mtt, &srq->buf); 329 if (ret) 330 goto err_srq_mtt; 331 332 page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz; 333 ret = hns_roce_create_idx_que(ib_srq->pd, srq, page_shift); 334 if (ret) { 335 dev_err(hr_dev->dev, "Create idx queue fail(%d)!\n", 336 ret); 337 goto err_srq_mtt; 338 } 339 340 /* Init mtt table for idx_que */ 341 ret = hns_roce_mtt_init(hr_dev, srq->idx_que.idx_buf.npages, 342 srq->idx_que.idx_buf.page_shift, 343 &srq->idx_que.mtt); 344 if (ret) 345 goto err_create_idx; 346 347 /* Write buffer address into the mtt table */ 348 ret = hns_roce_buf_write_mtt(hr_dev, &srq->idx_que.mtt, 349 &srq->idx_que.idx_buf); 350 if (ret) 351 goto err_idx_buf; 352 353 srq->wrid = kvmalloc_array(srq->max, sizeof(u64), GFP_KERNEL); 354 if (!srq->wrid) { 355 ret = -ENOMEM; 356 goto err_idx_buf; 357 } 358 } 359 360 cqn = ib_srq_has_cq(srq_init_attr->srq_type) ? 361 to_hr_cq(srq_init_attr->ext.cq)->cqn : 0; 362 363 srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG; 364 365 ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(ib_srq->pd)->pdn, cqn, 0, 366 &srq->mtt, 0, srq); 367 if (ret) 368 goto err_wrid; 369 370 srq->event = hns_roce_ib_srq_event; 371 srq->ibsrq.ext.xrc.srq_num = srq->srqn; 372 resp.srqn = srq->srqn; 373 374 if (udata) { 375 if (ib_copy_to_udata(udata, &resp, 376 min(udata->outlen, sizeof(resp)))) { 377 ret = -EFAULT; 378 goto err_srqc_alloc; 379 } 380 } 381 382 return 0; 383 384 err_srqc_alloc: 385 hns_roce_srq_free(hr_dev, srq); 386 387 err_wrid: 388 kvfree(srq->wrid); 389 390 err_idx_buf: 391 hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); 392 393 err_idx_mtt: 394 if (udata) 395 ib_umem_release(srq->idx_que.umem); 396 397 err_create_idx: 398 hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, 399 &srq->idx_que.idx_buf); 400 kfree(srq->idx_que.bitmap); 401 402 err_srq_mtt: 403 hns_roce_mtt_cleanup(hr_dev, &srq->mtt); 404 405 err_buf: 406 if (udata) 407 ib_umem_release(srq->umem); 408 else 409 hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf); 410 411 return ret; 412 } 413 414 void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 415 { 416 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 417 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 418 419 hns_roce_srq_free(hr_dev, srq); 420 hns_roce_mtt_cleanup(hr_dev, &srq->mtt); 421 422 if (ibsrq->uobject) { 423 hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); 424 ib_umem_release(srq->idx_que.umem); 425 ib_umem_release(srq->umem); 426 } else { 427 kvfree(srq->wrid); 428 hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift, 429 &srq->buf); 430 } 431 } 432 433 int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev) 434 { 435 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 436 437 xa_init(&srq_table->xa); 438 439 return hns_roce_bitmap_init(&srq_table->bitmap, hr_dev->caps.num_srqs, 440 hr_dev->caps.num_srqs - 1, 441 hr_dev->caps.reserved_srqs, 0); 442 } 443 444 void hns_roce_cleanup_srq_table(struct hns_roce_dev *hr_dev) 445 { 446 hns_roce_bitmap_cleanup(&hr_dev->srq_table.bitmap); 447 } 448