1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/platform_device.h> 35 #include <linux/vmalloc.h> 36 #include <rdma/ib_umem.h> 37 #include "hns_roce_device.h" 38 #include "hns_roce_cmd.h" 39 #include "hns_roce_hem.h" 40 41 static u32 hw_index_to_key(unsigned long ind) 42 { 43 return (u32)(ind >> 24) | (ind << 8); 44 } 45 46 unsigned long key_to_hw_index(u32 key) 47 { 48 return (key << 24) | (key >> 8); 49 } 50 EXPORT_SYMBOL_GPL(key_to_hw_index); 51 52 static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev, 53 struct hns_roce_cmd_mailbox *mailbox, 54 unsigned long mpt_index) 55 { 56 return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0, 57 HNS_ROCE_CMD_SW2HW_MPT, 58 HNS_ROCE_CMD_TIMEOUT_MSECS); 59 } 60 61 int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, 62 struct hns_roce_cmd_mailbox *mailbox, 63 unsigned long mpt_index) 64 { 65 return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0, 66 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT, 67 HNS_ROCE_CMD_TIMEOUT_MSECS); 68 } 69 EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt); 70 71 static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order, 72 unsigned long *seg) 73 { 74 int o; 75 u32 m; 76 77 spin_lock(&buddy->lock); 78 79 for (o = order; o <= buddy->max_order; ++o) { 80 if (buddy->num_free[o]) { 81 m = 1 << (buddy->max_order - o); 82 *seg = find_first_bit(buddy->bits[o], m); 83 if (*seg < m) 84 goto found; 85 } 86 } 87 spin_unlock(&buddy->lock); 88 return -1; 89 90 found: 91 clear_bit(*seg, buddy->bits[o]); 92 --buddy->num_free[o]; 93 94 while (o > order) { 95 --o; 96 *seg <<= 1; 97 set_bit(*seg ^ 1, buddy->bits[o]); 98 ++buddy->num_free[o]; 99 } 100 101 spin_unlock(&buddy->lock); 102 103 *seg <<= order; 104 return 0; 105 } 106 107 static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg, 108 int order) 109 { 110 seg >>= order; 111 112 spin_lock(&buddy->lock); 113 114 while (test_bit(seg ^ 1, buddy->bits[order])) { 115 clear_bit(seg ^ 1, buddy->bits[order]); 116 --buddy->num_free[order]; 117 seg >>= 1; 118 ++order; 119 } 120 121 set_bit(seg, buddy->bits[order]); 122 ++buddy->num_free[order]; 123 124 spin_unlock(&buddy->lock); 125 } 126 127 static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order) 128 { 129 int i, s; 130 131 buddy->max_order = max_order; 132 spin_lock_init(&buddy->lock); 133 buddy->bits = kcalloc(buddy->max_order + 1, 134 sizeof(*buddy->bits), 135 GFP_KERNEL); 136 buddy->num_free = kcalloc(buddy->max_order + 1, 137 sizeof(*buddy->num_free), 138 GFP_KERNEL); 139 if (!buddy->bits || !buddy->num_free) 140 goto err_out; 141 142 for (i = 0; i <= buddy->max_order; ++i) { 143 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 144 buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL | 145 __GFP_NOWARN); 146 if (!buddy->bits[i]) { 147 buddy->bits[i] = vzalloc(array_size(s, sizeof(long))); 148 if (!buddy->bits[i]) 149 goto err_out_free; 150 } 151 } 152 153 set_bit(0, buddy->bits[buddy->max_order]); 154 buddy->num_free[buddy->max_order] = 1; 155 156 return 0; 157 158 err_out_free: 159 for (i = 0; i <= buddy->max_order; ++i) 160 kvfree(buddy->bits[i]); 161 162 err_out: 163 kfree(buddy->bits); 164 kfree(buddy->num_free); 165 return -ENOMEM; 166 } 167 168 static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy) 169 { 170 int i; 171 172 for (i = 0; i <= buddy->max_order; ++i) 173 kvfree(buddy->bits[i]); 174 175 kfree(buddy->bits); 176 kfree(buddy->num_free); 177 } 178 179 static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order, 180 unsigned long *seg, u32 mtt_type) 181 { 182 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 183 struct hns_roce_hem_table *table; 184 struct hns_roce_buddy *buddy; 185 int ret; 186 187 if (mtt_type == MTT_TYPE_WQE) { 188 buddy = &mr_table->mtt_buddy; 189 table = &mr_table->mtt_table; 190 } else { 191 buddy = &mr_table->mtt_cqe_buddy; 192 table = &mr_table->mtt_cqe_table; 193 } 194 195 ret = hns_roce_buddy_alloc(buddy, order, seg); 196 if (ret == -1) 197 return -1; 198 199 if (hns_roce_table_get_range(hr_dev, table, *seg, 200 *seg + (1 << order) - 1)) { 201 hns_roce_buddy_free(buddy, *seg, order); 202 return -1; 203 } 204 205 return 0; 206 } 207 208 int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift, 209 struct hns_roce_mtt *mtt) 210 { 211 int ret; 212 int i; 213 214 /* Page num is zero, correspond to DMA memory register */ 215 if (!npages) { 216 mtt->order = -1; 217 mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT; 218 return 0; 219 } 220 221 /* Note: if page_shift is zero, FAST memory register */ 222 mtt->page_shift = page_shift; 223 224 /* Compute MTT entry necessary */ 225 for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages; 226 i <<= 1) 227 ++mtt->order; 228 229 /* Allocate MTT entry */ 230 ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg, 231 mtt->mtt_type); 232 if (ret == -1) 233 return -ENOMEM; 234 235 return 0; 236 } 237 238 void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt) 239 { 240 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 241 242 if (mtt->order < 0) 243 return; 244 245 if (mtt->mtt_type == MTT_TYPE_WQE) { 246 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, 247 mtt->order); 248 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table, 249 mtt->first_seg, 250 mtt->first_seg + (1 << mtt->order) - 1); 251 } else { 252 hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg, 253 mtt->order); 254 hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table, 255 mtt->first_seg, 256 mtt->first_seg + (1 << mtt->order) - 1); 257 } 258 } 259 EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup); 260 261 static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, 262 struct hns_roce_mr *mr, int err_loop_index, 263 int loop_i, int loop_j) 264 { 265 struct device *dev = hr_dev->dev; 266 u32 mhop_num; 267 u32 pbl_bt_sz; 268 u64 bt_idx; 269 int i, j; 270 271 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); 272 mhop_num = hr_dev->caps.pbl_hop_num; 273 274 i = loop_i; 275 if (mhop_num == 3 && err_loop_index == 2) { 276 for (; i >= 0; i--) { 277 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 278 mr->pbl_l1_dma_addr[i]); 279 280 for (j = 0; j < pbl_bt_sz / 8; j++) { 281 if (i == loop_i && j >= loop_j) 282 break; 283 284 bt_idx = i * pbl_bt_sz / 8 + j; 285 dma_free_coherent(dev, pbl_bt_sz, 286 mr->pbl_bt_l2[bt_idx], 287 mr->pbl_l2_dma_addr[bt_idx]); 288 } 289 } 290 } else if (mhop_num == 3 && err_loop_index == 1) { 291 for (i -= 1; i >= 0; i--) { 292 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 293 mr->pbl_l1_dma_addr[i]); 294 295 for (j = 0; j < pbl_bt_sz / 8; j++) { 296 bt_idx = i * pbl_bt_sz / 8 + j; 297 dma_free_coherent(dev, pbl_bt_sz, 298 mr->pbl_bt_l2[bt_idx], 299 mr->pbl_l2_dma_addr[bt_idx]); 300 } 301 } 302 } else if (mhop_num == 2 && err_loop_index == 1) { 303 for (i -= 1; i >= 0; i--) 304 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 305 mr->pbl_l1_dma_addr[i]); 306 } else { 307 dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.", 308 mhop_num, err_loop_index); 309 return; 310 } 311 312 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, mr->pbl_l0_dma_addr); 313 mr->pbl_bt_l0 = NULL; 314 mr->pbl_l0_dma_addr = 0; 315 } 316 317 /* PBL multi hop addressing */ 318 static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages, 319 struct hns_roce_mr *mr) 320 { 321 struct device *dev = hr_dev->dev; 322 int mr_alloc_done = 0; 323 int npages_allocated; 324 int i = 0, j = 0; 325 u32 pbl_bt_sz; 326 u32 mhop_num; 327 u64 pbl_last_bt_num; 328 u64 pbl_bt_cnt = 0; 329 u64 bt_idx; 330 u64 size; 331 332 mhop_num = (mr->type == MR_TYPE_FRMR ? 1 : hr_dev->caps.pbl_hop_num); 333 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); 334 pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8); 335 336 if (mhop_num == HNS_ROCE_HOP_NUM_0) 337 return 0; 338 339 /* hop_num = 1 */ 340 if (mhop_num == 1) { 341 if (npages > pbl_bt_sz / 8) { 342 dev_err(dev, "npages %d is larger than buf_pg_sz!", 343 npages); 344 return -EINVAL; 345 } 346 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, 347 &(mr->pbl_dma_addr), 348 GFP_KERNEL); 349 if (!mr->pbl_buf) 350 return -ENOMEM; 351 352 mr->pbl_size = npages; 353 mr->pbl_ba = mr->pbl_dma_addr; 354 mr->pbl_hop_num = mhop_num; 355 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; 356 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; 357 return 0; 358 } 359 360 mr->pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8, 361 sizeof(*mr->pbl_l1_dma_addr), 362 GFP_KERNEL); 363 if (!mr->pbl_l1_dma_addr) 364 return -ENOMEM; 365 366 mr->pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr->pbl_bt_l1), 367 GFP_KERNEL); 368 if (!mr->pbl_bt_l1) 369 goto err_kcalloc_bt_l1; 370 371 if (mhop_num == 3) { 372 mr->pbl_l2_dma_addr = kcalloc(pbl_last_bt_num, 373 sizeof(*mr->pbl_l2_dma_addr), 374 GFP_KERNEL); 375 if (!mr->pbl_l2_dma_addr) 376 goto err_kcalloc_l2_dma; 377 378 mr->pbl_bt_l2 = kcalloc(pbl_last_bt_num, 379 sizeof(*mr->pbl_bt_l2), 380 GFP_KERNEL); 381 if (!mr->pbl_bt_l2) 382 goto err_kcalloc_bt_l2; 383 } 384 385 /* alloc L0 BT */ 386 mr->pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz, 387 &(mr->pbl_l0_dma_addr), 388 GFP_KERNEL); 389 if (!mr->pbl_bt_l0) 390 goto err_dma_alloc_l0; 391 392 if (mhop_num == 2) { 393 /* alloc L1 BT */ 394 for (i = 0; i < pbl_bt_sz / 8; i++) { 395 if (pbl_bt_cnt + 1 < pbl_last_bt_num) { 396 size = pbl_bt_sz; 397 } else { 398 npages_allocated = i * (pbl_bt_sz / 8); 399 size = (npages - npages_allocated) * 8; 400 } 401 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, size, 402 &(mr->pbl_l1_dma_addr[i]), 403 GFP_KERNEL); 404 if (!mr->pbl_bt_l1[i]) { 405 hns_roce_loop_free(hr_dev, mr, 1, i, 0); 406 goto err_dma_alloc_l0; 407 } 408 409 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i]; 410 411 pbl_bt_cnt++; 412 if (pbl_bt_cnt >= pbl_last_bt_num) 413 break; 414 } 415 } else if (mhop_num == 3) { 416 /* alloc L1, L2 BT */ 417 for (i = 0; i < pbl_bt_sz / 8; i++) { 418 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz, 419 &(mr->pbl_l1_dma_addr[i]), 420 GFP_KERNEL); 421 if (!mr->pbl_bt_l1[i]) { 422 hns_roce_loop_free(hr_dev, mr, 1, i, 0); 423 goto err_dma_alloc_l0; 424 } 425 426 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i]; 427 428 for (j = 0; j < pbl_bt_sz / 8; j++) { 429 bt_idx = i * pbl_bt_sz / 8 + j; 430 431 if (pbl_bt_cnt + 1 < pbl_last_bt_num) { 432 size = pbl_bt_sz; 433 } else { 434 npages_allocated = bt_idx * 435 (pbl_bt_sz / 8); 436 size = (npages - npages_allocated) * 8; 437 } 438 mr->pbl_bt_l2[bt_idx] = dma_alloc_coherent( 439 dev, size, 440 &(mr->pbl_l2_dma_addr[bt_idx]), 441 GFP_KERNEL); 442 if (!mr->pbl_bt_l2[bt_idx]) { 443 hns_roce_loop_free(hr_dev, mr, 2, i, j); 444 goto err_dma_alloc_l0; 445 } 446 447 *(mr->pbl_bt_l1[i] + j) = 448 mr->pbl_l2_dma_addr[bt_idx]; 449 450 pbl_bt_cnt++; 451 if (pbl_bt_cnt >= pbl_last_bt_num) { 452 mr_alloc_done = 1; 453 break; 454 } 455 } 456 457 if (mr_alloc_done) 458 break; 459 } 460 } 461 462 mr->l0_chunk_last_num = i + 1; 463 if (mhop_num == 3) 464 mr->l1_chunk_last_num = j + 1; 465 466 mr->pbl_size = npages; 467 mr->pbl_ba = mr->pbl_l0_dma_addr; 468 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num; 469 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; 470 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; 471 472 return 0; 473 474 err_dma_alloc_l0: 475 kfree(mr->pbl_bt_l2); 476 mr->pbl_bt_l2 = NULL; 477 478 err_kcalloc_bt_l2: 479 kfree(mr->pbl_l2_dma_addr); 480 mr->pbl_l2_dma_addr = NULL; 481 482 err_kcalloc_l2_dma: 483 kfree(mr->pbl_bt_l1); 484 mr->pbl_bt_l1 = NULL; 485 486 err_kcalloc_bt_l1: 487 kfree(mr->pbl_l1_dma_addr); 488 mr->pbl_l1_dma_addr = NULL; 489 490 return -ENOMEM; 491 } 492 493 static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, 494 u64 size, u32 access, int npages, 495 struct hns_roce_mr *mr) 496 { 497 struct device *dev = hr_dev->dev; 498 unsigned long index = 0; 499 int ret = 0; 500 501 /* Allocate a key for mr from mr_table */ 502 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); 503 if (ret == -1) 504 return -ENOMEM; 505 506 mr->iova = iova; /* MR va starting addr */ 507 mr->size = size; /* MR addr range */ 508 mr->pd = pd; /* MR num */ 509 mr->access = access; /* MR access permit */ 510 mr->enabled = 0; /* MR active status */ 511 mr->key = hw_index_to_key(index); /* MR key */ 512 513 if (size == ~0ull) { 514 mr->pbl_buf = NULL; 515 mr->pbl_dma_addr = 0; 516 /* PBL multi-hop addressing parameters */ 517 mr->pbl_bt_l2 = NULL; 518 mr->pbl_bt_l1 = NULL; 519 mr->pbl_bt_l0 = NULL; 520 mr->pbl_l2_dma_addr = NULL; 521 mr->pbl_l1_dma_addr = NULL; 522 mr->pbl_l0_dma_addr = 0; 523 } else { 524 if (!hr_dev->caps.pbl_hop_num) { 525 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, 526 &(mr->pbl_dma_addr), 527 GFP_KERNEL); 528 if (!mr->pbl_buf) 529 return -ENOMEM; 530 } else { 531 ret = hns_roce_mhop_alloc(hr_dev, npages, mr); 532 } 533 } 534 535 return ret; 536 } 537 538 static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, 539 struct hns_roce_mr *mr) 540 { 541 struct device *dev = hr_dev->dev; 542 int npages_allocated; 543 int npages; 544 int i, j; 545 u32 pbl_bt_sz; 546 u32 mhop_num; 547 u64 bt_idx; 548 549 npages = mr->pbl_size; 550 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); 551 mhop_num = (mr->type == MR_TYPE_FRMR) ? 1 : hr_dev->caps.pbl_hop_num; 552 553 if (mhop_num == HNS_ROCE_HOP_NUM_0) 554 return; 555 556 /* hop_num = 1 */ 557 if (mhop_num == 1) { 558 dma_free_coherent(dev, (unsigned int)(npages * 8), 559 mr->pbl_buf, mr->pbl_dma_addr); 560 return; 561 } 562 563 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, 564 mr->pbl_l0_dma_addr); 565 566 if (mhop_num == 2) { 567 for (i = 0; i < mr->l0_chunk_last_num; i++) { 568 if (i == mr->l0_chunk_last_num - 1) { 569 npages_allocated = i * (pbl_bt_sz / 8); 570 571 dma_free_coherent(dev, 572 (npages - npages_allocated) * 8, 573 mr->pbl_bt_l1[i], 574 mr->pbl_l1_dma_addr[i]); 575 576 break; 577 } 578 579 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 580 mr->pbl_l1_dma_addr[i]); 581 } 582 } else if (mhop_num == 3) { 583 for (i = 0; i < mr->l0_chunk_last_num; i++) { 584 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], 585 mr->pbl_l1_dma_addr[i]); 586 587 for (j = 0; j < pbl_bt_sz / 8; j++) { 588 bt_idx = i * (pbl_bt_sz / 8) + j; 589 590 if ((i == mr->l0_chunk_last_num - 1) 591 && j == mr->l1_chunk_last_num - 1) { 592 npages_allocated = bt_idx * 593 (pbl_bt_sz / 8); 594 595 dma_free_coherent(dev, 596 (npages - npages_allocated) * 8, 597 mr->pbl_bt_l2[bt_idx], 598 mr->pbl_l2_dma_addr[bt_idx]); 599 600 break; 601 } 602 603 dma_free_coherent(dev, pbl_bt_sz, 604 mr->pbl_bt_l2[bt_idx], 605 mr->pbl_l2_dma_addr[bt_idx]); 606 } 607 } 608 } 609 610 kfree(mr->pbl_bt_l1); 611 kfree(mr->pbl_l1_dma_addr); 612 mr->pbl_bt_l1 = NULL; 613 mr->pbl_l1_dma_addr = NULL; 614 if (mhop_num == 3) { 615 kfree(mr->pbl_bt_l2); 616 kfree(mr->pbl_l2_dma_addr); 617 mr->pbl_bt_l2 = NULL; 618 mr->pbl_l2_dma_addr = NULL; 619 } 620 } 621 622 static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, 623 struct hns_roce_mr *mr) 624 { 625 struct device *dev = hr_dev->dev; 626 int npages = 0; 627 int ret; 628 629 if (mr->enabled) { 630 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key) 631 & (hr_dev->caps.num_mtpts - 1)); 632 if (ret) 633 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret); 634 } 635 636 if (mr->size != ~0ULL) { 637 if (mr->type == MR_TYPE_MR) 638 npages = ib_umem_page_count(mr->umem); 639 640 if (!hr_dev->caps.pbl_hop_num) 641 dma_free_coherent(dev, (unsigned int)(npages * 8), 642 mr->pbl_buf, mr->pbl_dma_addr); 643 else 644 hns_roce_mhop_free(hr_dev, mr); 645 } 646 647 if (mr->enabled) 648 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, 649 key_to_hw_index(mr->key)); 650 651 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, 652 key_to_hw_index(mr->key), BITMAP_NO_RR); 653 } 654 655 static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, 656 struct hns_roce_mr *mr) 657 { 658 int ret; 659 unsigned long mtpt_idx = key_to_hw_index(mr->key); 660 struct device *dev = hr_dev->dev; 661 struct hns_roce_cmd_mailbox *mailbox; 662 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 663 664 /* Prepare HEM entry memory */ 665 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx); 666 if (ret) 667 return ret; 668 669 /* Allocate mailbox memory */ 670 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 671 if (IS_ERR(mailbox)) { 672 ret = PTR_ERR(mailbox); 673 goto err_table; 674 } 675 676 if (mr->type != MR_TYPE_FRMR) 677 ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx); 678 else 679 ret = hr_dev->hw->frmr_write_mtpt(mailbox->buf, mr); 680 if (ret) { 681 dev_err(dev, "Write mtpt fail!\n"); 682 goto err_page; 683 } 684 685 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, 686 mtpt_idx & (hr_dev->caps.num_mtpts - 1)); 687 if (ret) { 688 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret); 689 goto err_page; 690 } 691 692 mr->enabled = 1; 693 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 694 695 return 0; 696 697 err_page: 698 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 699 700 err_table: 701 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx); 702 return ret; 703 } 704 705 static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, 706 struct hns_roce_mtt *mtt, u32 start_index, 707 u32 npages, u64 *page_list) 708 { 709 struct hns_roce_hem_table *table; 710 dma_addr_t dma_handle; 711 __le64 *mtts; 712 u32 s = start_index * sizeof(u64); 713 u32 bt_page_size; 714 u32 i; 715 716 if (mtt->mtt_type == MTT_TYPE_WQE) 717 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); 718 else 719 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); 720 721 /* All MTTs must fit in the same page */ 722 if (start_index / (bt_page_size / sizeof(u64)) != 723 (start_index + npages - 1) / (bt_page_size / sizeof(u64))) 724 return -EINVAL; 725 726 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1)) 727 return -EINVAL; 728 729 if (mtt->mtt_type == MTT_TYPE_WQE) 730 table = &hr_dev->mr_table.mtt_table; 731 else 732 table = &hr_dev->mr_table.mtt_cqe_table; 733 734 mtts = hns_roce_table_find(hr_dev, table, 735 mtt->first_seg + s / hr_dev->caps.mtt_entry_sz, 736 &dma_handle); 737 if (!mtts) 738 return -ENOMEM; 739 740 /* Save page addr, low 12 bits : 0 */ 741 for (i = 0; i < npages; ++i) { 742 if (!hr_dev->caps.mtt_hop_num) 743 mtts[i] = cpu_to_le64(page_list[i] >> PAGE_ADDR_SHIFT); 744 else 745 mtts[i] = cpu_to_le64(page_list[i]); 746 } 747 748 return 0; 749 } 750 751 static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev, 752 struct hns_roce_mtt *mtt, u32 start_index, 753 u32 npages, u64 *page_list) 754 { 755 int chunk; 756 int ret; 757 u32 bt_page_size; 758 759 if (mtt->order < 0) 760 return -EINVAL; 761 762 if (mtt->mtt_type == MTT_TYPE_WQE) 763 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); 764 else 765 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); 766 767 while (npages > 0) { 768 chunk = min_t(int, bt_page_size / sizeof(u64), npages); 769 770 ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk, 771 page_list); 772 if (ret) 773 return ret; 774 775 npages -= chunk; 776 start_index += chunk; 777 page_list += chunk; 778 } 779 780 return 0; 781 } 782 783 int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, 784 struct hns_roce_mtt *mtt, struct hns_roce_buf *buf) 785 { 786 u64 *page_list; 787 int ret; 788 u32 i; 789 790 page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL); 791 if (!page_list) 792 return -ENOMEM; 793 794 for (i = 0; i < buf->npages; ++i) { 795 if (buf->nbufs == 1) 796 page_list[i] = buf->direct.map + (i << buf->page_shift); 797 else 798 page_list[i] = buf->page_list[i].map; 799 800 } 801 ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list); 802 803 kfree(page_list); 804 805 return ret; 806 } 807 808 int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) 809 { 810 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 811 int ret; 812 813 ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap, 814 hr_dev->caps.num_mtpts, 815 hr_dev->caps.num_mtpts - 1, 816 hr_dev->caps.reserved_mrws, 0); 817 if (ret) 818 return ret; 819 820 ret = hns_roce_buddy_init(&mr_table->mtt_buddy, 821 ilog2(hr_dev->caps.num_mtt_segs)); 822 if (ret) 823 goto err_buddy; 824 825 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) { 826 ret = hns_roce_buddy_init(&mr_table->mtt_cqe_buddy, 827 ilog2(hr_dev->caps.num_cqe_segs)); 828 if (ret) 829 goto err_buddy_cqe; 830 } 831 return 0; 832 833 err_buddy_cqe: 834 hns_roce_buddy_cleanup(&mr_table->mtt_buddy); 835 836 err_buddy: 837 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); 838 return ret; 839 } 840 841 void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev) 842 { 843 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 844 845 hns_roce_buddy_cleanup(&mr_table->mtt_buddy); 846 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) 847 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy); 848 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); 849 } 850 851 struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc) 852 { 853 struct hns_roce_mr *mr; 854 int ret; 855 856 mr = kmalloc(sizeof(*mr), GFP_KERNEL); 857 if (mr == NULL) 858 return ERR_PTR(-ENOMEM); 859 860 mr->type = MR_TYPE_DMA; 861 862 /* Allocate memory region key */ 863 ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0, 864 ~0ULL, acc, 0, mr); 865 if (ret) 866 goto err_free; 867 868 ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr); 869 if (ret) 870 goto err_mr; 871 872 mr->ibmr.rkey = mr->ibmr.lkey = mr->key; 873 mr->umem = NULL; 874 875 return &mr->ibmr; 876 877 err_mr: 878 hns_roce_mr_free(to_hr_dev(pd->device), mr); 879 880 err_free: 881 kfree(mr); 882 return ERR_PTR(ret); 883 } 884 885 int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, 886 struct hns_roce_mtt *mtt, struct ib_umem *umem) 887 { 888 struct device *dev = hr_dev->dev; 889 struct scatterlist *sg; 890 unsigned int order; 891 int i, k, entry; 892 int npage = 0; 893 int ret = 0; 894 int len; 895 u64 page_addr; 896 u64 *pages; 897 u32 bt_page_size; 898 u32 n; 899 900 order = mtt->mtt_type == MTT_TYPE_WQE ? hr_dev->caps.mtt_ba_pg_sz : 901 hr_dev->caps.cqe_ba_pg_sz; 902 bt_page_size = 1 << (order + PAGE_SHIFT); 903 904 pages = (u64 *) __get_free_pages(GFP_KERNEL, order); 905 if (!pages) 906 return -ENOMEM; 907 908 i = n = 0; 909 910 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 911 len = sg_dma_len(sg) >> PAGE_SHIFT; 912 for (k = 0; k < len; ++k) { 913 page_addr = 914 sg_dma_address(sg) + (k << umem->page_shift); 915 if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) { 916 if (page_addr & ((1 << mtt->page_shift) - 1)) { 917 dev_err(dev, "page_addr 0x%llx is not page_shift %d alignment!\n", 918 page_addr, mtt->page_shift); 919 ret = -EINVAL; 920 goto out; 921 } 922 pages[i++] = page_addr; 923 } 924 npage++; 925 if (i == bt_page_size / sizeof(u64)) { 926 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, 927 pages); 928 if (ret) 929 goto out; 930 n += i; 931 i = 0; 932 } 933 } 934 } 935 936 if (i) 937 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages); 938 939 out: 940 free_pages((unsigned long) pages, order); 941 return ret; 942 } 943 944 static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev, 945 struct hns_roce_mr *mr, 946 struct ib_umem *umem) 947 { 948 struct scatterlist *sg; 949 int i = 0, j = 0, k; 950 int entry; 951 int len; 952 u64 page_addr; 953 u32 pbl_bt_sz; 954 955 if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0) 956 return 0; 957 958 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT); 959 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 960 len = sg_dma_len(sg) >> PAGE_SHIFT; 961 for (k = 0; k < len; ++k) { 962 page_addr = sg_dma_address(sg) + 963 (k << umem->page_shift); 964 965 if (!hr_dev->caps.pbl_hop_num) { 966 mr->pbl_buf[i++] = page_addr >> 12; 967 } else if (hr_dev->caps.pbl_hop_num == 1) { 968 mr->pbl_buf[i++] = page_addr; 969 } else { 970 if (hr_dev->caps.pbl_hop_num == 2) 971 mr->pbl_bt_l1[i][j] = page_addr; 972 else if (hr_dev->caps.pbl_hop_num == 3) 973 mr->pbl_bt_l2[i][j] = page_addr; 974 975 j++; 976 if (j >= (pbl_bt_sz / 8)) { 977 i++; 978 j = 0; 979 } 980 } 981 } 982 } 983 984 /* Memory barrier */ 985 mb(); 986 987 return 0; 988 } 989 990 struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 991 u64 virt_addr, int access_flags, 992 struct ib_udata *udata) 993 { 994 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 995 struct device *dev = hr_dev->dev; 996 struct hns_roce_mr *mr; 997 int bt_size; 998 int ret; 999 int n; 1000 int i; 1001 1002 mr = kmalloc(sizeof(*mr), GFP_KERNEL); 1003 if (!mr) 1004 return ERR_PTR(-ENOMEM); 1005 1006 mr->umem = ib_umem_get(pd->uobject->context, start, length, 1007 access_flags, 0); 1008 if (IS_ERR(mr->umem)) { 1009 ret = PTR_ERR(mr->umem); 1010 goto err_free; 1011 } 1012 1013 n = ib_umem_page_count(mr->umem); 1014 1015 if (!hr_dev->caps.pbl_hop_num) { 1016 if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) { 1017 dev_err(dev, 1018 " MR len %lld err. MR is limited to 4G at most!\n", 1019 length); 1020 ret = -EINVAL; 1021 goto err_umem; 1022 } 1023 } else { 1024 int pbl_size = 1; 1025 1026 bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8; 1027 for (i = 0; i < hr_dev->caps.pbl_hop_num; i++) 1028 pbl_size *= bt_size; 1029 if (n > pbl_size) { 1030 dev_err(dev, 1031 " MR len %lld err. MR page num is limited to %d!\n", 1032 length, pbl_size); 1033 ret = -EINVAL; 1034 goto err_umem; 1035 } 1036 } 1037 1038 mr->type = MR_TYPE_MR; 1039 1040 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length, 1041 access_flags, n, mr); 1042 if (ret) 1043 goto err_umem; 1044 1045 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem); 1046 if (ret) 1047 goto err_mr; 1048 1049 ret = hns_roce_mr_enable(hr_dev, mr); 1050 if (ret) 1051 goto err_mr; 1052 1053 mr->ibmr.rkey = mr->ibmr.lkey = mr->key; 1054 1055 return &mr->ibmr; 1056 1057 err_mr: 1058 hns_roce_mr_free(hr_dev, mr); 1059 1060 err_umem: 1061 ib_umem_release(mr->umem); 1062 1063 err_free: 1064 kfree(mr); 1065 return ERR_PTR(ret); 1066 } 1067 1068 int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length, 1069 u64 virt_addr, int mr_access_flags, struct ib_pd *pd, 1070 struct ib_udata *udata) 1071 { 1072 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); 1073 struct hns_roce_mr *mr = to_hr_mr(ibmr); 1074 struct hns_roce_cmd_mailbox *mailbox; 1075 struct device *dev = hr_dev->dev; 1076 unsigned long mtpt_idx; 1077 u32 pdn = 0; 1078 int npages; 1079 int ret; 1080 1081 if (!mr->enabled) 1082 return -EINVAL; 1083 1084 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 1085 if (IS_ERR(mailbox)) 1086 return PTR_ERR(mailbox); 1087 1088 mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1); 1089 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0, 1090 HNS_ROCE_CMD_QUERY_MPT, 1091 HNS_ROCE_CMD_TIMEOUT_MSECS); 1092 if (ret) 1093 goto free_cmd_mbox; 1094 1095 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx); 1096 if (ret) 1097 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret); 1098 1099 mr->enabled = 0; 1100 1101 if (flags & IB_MR_REREG_PD) 1102 pdn = to_hr_pd(pd)->pdn; 1103 1104 if (flags & IB_MR_REREG_TRANS) { 1105 if (mr->size != ~0ULL) { 1106 npages = ib_umem_page_count(mr->umem); 1107 1108 if (hr_dev->caps.pbl_hop_num) 1109 hns_roce_mhop_free(hr_dev, mr); 1110 else 1111 dma_free_coherent(dev, npages * 8, mr->pbl_buf, 1112 mr->pbl_dma_addr); 1113 } 1114 ib_umem_release(mr->umem); 1115 1116 mr->umem = ib_umem_get(ibmr->uobject->context, start, length, 1117 mr_access_flags, 0); 1118 if (IS_ERR(mr->umem)) { 1119 ret = PTR_ERR(mr->umem); 1120 mr->umem = NULL; 1121 goto free_cmd_mbox; 1122 } 1123 npages = ib_umem_page_count(mr->umem); 1124 1125 if (hr_dev->caps.pbl_hop_num) { 1126 ret = hns_roce_mhop_alloc(hr_dev, npages, mr); 1127 if (ret) 1128 goto release_umem; 1129 } else { 1130 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, 1131 &(mr->pbl_dma_addr), 1132 GFP_KERNEL); 1133 if (!mr->pbl_buf) { 1134 ret = -ENOMEM; 1135 goto release_umem; 1136 } 1137 } 1138 } 1139 1140 ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn, 1141 mr_access_flags, virt_addr, 1142 length, mailbox->buf); 1143 if (ret) { 1144 if (flags & IB_MR_REREG_TRANS) 1145 goto release_umem; 1146 else 1147 goto free_cmd_mbox; 1148 } 1149 1150 if (flags & IB_MR_REREG_TRANS) { 1151 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem); 1152 if (ret) { 1153 if (mr->size != ~0ULL) { 1154 npages = ib_umem_page_count(mr->umem); 1155 1156 if (hr_dev->caps.pbl_hop_num) 1157 hns_roce_mhop_free(hr_dev, mr); 1158 else 1159 dma_free_coherent(dev, npages * 8, 1160 mr->pbl_buf, 1161 mr->pbl_dma_addr); 1162 } 1163 1164 goto release_umem; 1165 } 1166 } 1167 1168 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx); 1169 if (ret) { 1170 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret); 1171 goto release_umem; 1172 } 1173 1174 mr->enabled = 1; 1175 if (flags & IB_MR_REREG_ACCESS) 1176 mr->access = mr_access_flags; 1177 1178 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 1179 1180 return 0; 1181 1182 release_umem: 1183 ib_umem_release(mr->umem); 1184 1185 free_cmd_mbox: 1186 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 1187 1188 return ret; 1189 } 1190 1191 int hns_roce_dereg_mr(struct ib_mr *ibmr) 1192 { 1193 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); 1194 struct hns_roce_mr *mr = to_hr_mr(ibmr); 1195 int ret = 0; 1196 1197 if (hr_dev->hw->dereg_mr) { 1198 ret = hr_dev->hw->dereg_mr(hr_dev, mr); 1199 } else { 1200 hns_roce_mr_free(hr_dev, mr); 1201 1202 if (mr->umem) 1203 ib_umem_release(mr->umem); 1204 1205 kfree(mr); 1206 } 1207 1208 return ret; 1209 } 1210 1211 struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 1212 u32 max_num_sg) 1213 { 1214 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 1215 struct device *dev = hr_dev->dev; 1216 struct hns_roce_mr *mr; 1217 u64 length; 1218 u32 page_size; 1219 int ret; 1220 1221 page_size = 1 << (hr_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT); 1222 length = max_num_sg * page_size; 1223 1224 if (mr_type != IB_MR_TYPE_MEM_REG) 1225 return ERR_PTR(-EINVAL); 1226 1227 if (max_num_sg > HNS_ROCE_FRMR_MAX_PA) { 1228 dev_err(dev, "max_num_sg larger than %d\n", 1229 HNS_ROCE_FRMR_MAX_PA); 1230 return ERR_PTR(-EINVAL); 1231 } 1232 1233 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1234 if (!mr) 1235 return ERR_PTR(-ENOMEM); 1236 1237 mr->type = MR_TYPE_FRMR; 1238 1239 /* Allocate memory region key */ 1240 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, 0, length, 1241 0, max_num_sg, mr); 1242 if (ret) 1243 goto err_free; 1244 1245 ret = hns_roce_mr_enable(hr_dev, mr); 1246 if (ret) 1247 goto err_mr; 1248 1249 mr->ibmr.rkey = mr->ibmr.lkey = mr->key; 1250 mr->umem = NULL; 1251 1252 return &mr->ibmr; 1253 1254 err_mr: 1255 hns_roce_mr_free(to_hr_dev(pd->device), mr); 1256 1257 err_free: 1258 kfree(mr); 1259 return ERR_PTR(ret); 1260 } 1261 1262 static int hns_roce_set_page(struct ib_mr *ibmr, u64 addr) 1263 { 1264 struct hns_roce_mr *mr = to_hr_mr(ibmr); 1265 1266 mr->pbl_buf[mr->npages++] = cpu_to_le64(addr); 1267 1268 return 0; 1269 } 1270 1271 int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1272 unsigned int *sg_offset) 1273 { 1274 struct hns_roce_mr *mr = to_hr_mr(ibmr); 1275 1276 mr->npages = 0; 1277 1278 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page); 1279 } 1280 1281 static void hns_roce_mw_free(struct hns_roce_dev *hr_dev, 1282 struct hns_roce_mw *mw) 1283 { 1284 struct device *dev = hr_dev->dev; 1285 int ret; 1286 1287 if (mw->enabled) { 1288 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mw->rkey) 1289 & (hr_dev->caps.num_mtpts - 1)); 1290 if (ret) 1291 dev_warn(dev, "MW HW2SW_MPT failed (%d)\n", ret); 1292 1293 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, 1294 key_to_hw_index(mw->rkey)); 1295 } 1296 1297 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, 1298 key_to_hw_index(mw->rkey), BITMAP_NO_RR); 1299 } 1300 1301 static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev, 1302 struct hns_roce_mw *mw) 1303 { 1304 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 1305 struct hns_roce_cmd_mailbox *mailbox; 1306 struct device *dev = hr_dev->dev; 1307 unsigned long mtpt_idx = key_to_hw_index(mw->rkey); 1308 int ret; 1309 1310 /* prepare HEM entry memory */ 1311 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx); 1312 if (ret) 1313 return ret; 1314 1315 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 1316 if (IS_ERR(mailbox)) { 1317 ret = PTR_ERR(mailbox); 1318 goto err_table; 1319 } 1320 1321 ret = hr_dev->hw->mw_write_mtpt(mailbox->buf, mw); 1322 if (ret) { 1323 dev_err(dev, "MW write mtpt fail!\n"); 1324 goto err_page; 1325 } 1326 1327 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, 1328 mtpt_idx & (hr_dev->caps.num_mtpts - 1)); 1329 if (ret) { 1330 dev_err(dev, "MW sw2hw_mpt failed (%d)\n", ret); 1331 goto err_page; 1332 } 1333 1334 mw->enabled = 1; 1335 1336 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 1337 1338 return 0; 1339 1340 err_page: 1341 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 1342 1343 err_table: 1344 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx); 1345 1346 return ret; 1347 } 1348 1349 struct ib_mw *hns_roce_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 1350 struct ib_udata *udata) 1351 { 1352 struct hns_roce_dev *hr_dev = to_hr_dev(ib_pd->device); 1353 struct hns_roce_mw *mw; 1354 unsigned long index = 0; 1355 int ret; 1356 1357 mw = kmalloc(sizeof(*mw), GFP_KERNEL); 1358 if (!mw) 1359 return ERR_PTR(-ENOMEM); 1360 1361 /* Allocate a key for mw from bitmap */ 1362 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); 1363 if (ret) 1364 goto err_bitmap; 1365 1366 mw->rkey = hw_index_to_key(index); 1367 1368 mw->ibmw.rkey = mw->rkey; 1369 mw->ibmw.type = type; 1370 mw->pdn = to_hr_pd(ib_pd)->pdn; 1371 mw->pbl_hop_num = hr_dev->caps.pbl_hop_num; 1372 mw->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz; 1373 mw->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz; 1374 1375 ret = hns_roce_mw_enable(hr_dev, mw); 1376 if (ret) 1377 goto err_mw; 1378 1379 return &mw->ibmw; 1380 1381 err_mw: 1382 hns_roce_mw_free(hr_dev, mw); 1383 1384 err_bitmap: 1385 kfree(mw); 1386 1387 return ERR_PTR(ret); 1388 } 1389 1390 int hns_roce_dealloc_mw(struct ib_mw *ibmw) 1391 { 1392 struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device); 1393 struct hns_roce_mw *mw = to_hr_mw(ibmw); 1394 1395 hns_roce_mw_free(hr_dev, mw); 1396 kfree(mw); 1397 1398 return 0; 1399 } 1400