1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/platform_device.h> 35 #include <rdma/ib_umem.h> 36 #include "hns_roce_device.h" 37 #include "hns_roce_cmd.h" 38 #include "hns_roce_hem.h" 39 40 static u32 hw_index_to_key(unsigned long ind) 41 { 42 return (u32)(ind >> 24) | (ind << 8); 43 } 44 45 unsigned long key_to_hw_index(u32 key) 46 { 47 return (key << 24) | (key >> 8); 48 } 49 50 static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev, 51 struct hns_roce_cmd_mailbox *mailbox, 52 unsigned long mpt_index) 53 { 54 return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0, 55 HNS_ROCE_CMD_SW2HW_MPT, 56 HNS_ROCE_CMD_TIMEOUT_MSECS); 57 } 58 59 int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, 60 struct hns_roce_cmd_mailbox *mailbox, 61 unsigned long mpt_index) 62 { 63 return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0, 64 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT, 65 HNS_ROCE_CMD_TIMEOUT_MSECS); 66 } 67 68 static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order, 69 unsigned long *seg) 70 { 71 int o; 72 u32 m; 73 74 spin_lock(&buddy->lock); 75 76 for (o = order; o <= buddy->max_order; ++o) { 77 if (buddy->num_free[o]) { 78 m = 1 << (buddy->max_order - o); 79 *seg = find_first_bit(buddy->bits[o], m); 80 if (*seg < m) 81 goto found; 82 } 83 } 84 spin_unlock(&buddy->lock); 85 return -1; 86 87 found: 88 clear_bit(*seg, buddy->bits[o]); 89 --buddy->num_free[o]; 90 91 while (o > order) { 92 --o; 93 *seg <<= 1; 94 set_bit(*seg ^ 1, buddy->bits[o]); 95 ++buddy->num_free[o]; 96 } 97 98 spin_unlock(&buddy->lock); 99 100 *seg <<= order; 101 return 0; 102 } 103 104 static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg, 105 int order) 106 { 107 seg >>= order; 108 109 spin_lock(&buddy->lock); 110 111 while (test_bit(seg ^ 1, buddy->bits[order])) { 112 clear_bit(seg ^ 1, buddy->bits[order]); 113 --buddy->num_free[order]; 114 seg >>= 1; 115 ++order; 116 } 117 118 set_bit(seg, buddy->bits[order]); 119 ++buddy->num_free[order]; 120 121 spin_unlock(&buddy->lock); 122 } 123 124 static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order) 125 { 126 int i, s; 127 128 buddy->max_order = max_order; 129 spin_lock_init(&buddy->lock); 130 131 buddy->bits = kzalloc((buddy->max_order + 1) * sizeof(long *), 132 GFP_KERNEL); 133 buddy->num_free = kzalloc((buddy->max_order + 1) * sizeof(int *), 134 GFP_KERNEL); 135 if (!buddy->bits || !buddy->num_free) 136 goto err_out; 137 138 for (i = 0; i <= buddy->max_order; ++i) { 139 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 140 buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL | 141 __GFP_NOWARN); 142 if (!buddy->bits[i]) { 143 buddy->bits[i] = vzalloc(s * sizeof(long)); 144 if (!buddy->bits[i]) 145 goto err_out_free; 146 } 147 } 148 149 set_bit(0, buddy->bits[buddy->max_order]); 150 buddy->num_free[buddy->max_order] = 1; 151 152 return 0; 153 154 err_out_free: 155 for (i = 0; i <= buddy->max_order; ++i) 156 kvfree(buddy->bits[i]); 157 158 err_out: 159 kfree(buddy->bits); 160 kfree(buddy->num_free); 161 return -ENOMEM; 162 } 163 164 static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy) 165 { 166 int i; 167 168 for (i = 0; i <= buddy->max_order; ++i) 169 kvfree(buddy->bits[i]); 170 171 kfree(buddy->bits); 172 kfree(buddy->num_free); 173 } 174 175 static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order, 176 unsigned long *seg) 177 { 178 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 179 int ret = 0; 180 181 ret = hns_roce_buddy_alloc(&mr_table->mtt_buddy, order, seg); 182 if (ret == -1) 183 return -1; 184 185 if (hns_roce_table_get_range(hr_dev, &mr_table->mtt_table, *seg, 186 *seg + (1 << order) - 1)) { 187 hns_roce_buddy_free(&mr_table->mtt_buddy, *seg, order); 188 return -1; 189 } 190 191 return 0; 192 } 193 194 int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift, 195 struct hns_roce_mtt *mtt) 196 { 197 int ret = 0; 198 int i; 199 200 /* Page num is zero, correspond to DMA memory register */ 201 if (!npages) { 202 mtt->order = -1; 203 mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT; 204 return 0; 205 } 206 207 /* Note: if page_shift is zero, FAST memory regsiter */ 208 mtt->page_shift = page_shift; 209 210 /* Compute MTT entry necessary */ 211 for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages; 212 i <<= 1) 213 ++mtt->order; 214 215 /* Allocate MTT entry */ 216 ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg); 217 if (ret == -1) 218 return -ENOMEM; 219 220 return 0; 221 } 222 223 void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt) 224 { 225 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 226 227 if (mtt->order < 0) 228 return; 229 230 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, mtt->order); 231 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table, mtt->first_seg, 232 mtt->first_seg + (1 << mtt->order) - 1); 233 } 234 235 static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, 236 u64 size, u32 access, int npages, 237 struct hns_roce_mr *mr) 238 { 239 unsigned long index = 0; 240 int ret = 0; 241 struct device *dev = &hr_dev->pdev->dev; 242 243 /* Allocate a key for mr from mr_table */ 244 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); 245 if (ret == -1) 246 return -ENOMEM; 247 248 mr->iova = iova; /* MR va starting addr */ 249 mr->size = size; /* MR addr range */ 250 mr->pd = pd; /* MR num */ 251 mr->access = access; /* MR access permit */ 252 mr->enabled = 0; /* MR active status */ 253 mr->key = hw_index_to_key(index); /* MR key */ 254 255 if (size == ~0ull) { 256 mr->type = MR_TYPE_DMA; 257 mr->pbl_buf = NULL; 258 mr->pbl_dma_addr = 0; 259 } else { 260 mr->type = MR_TYPE_MR; 261 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, 262 &(mr->pbl_dma_addr), 263 GFP_KERNEL); 264 if (!mr->pbl_buf) 265 return -ENOMEM; 266 } 267 268 return 0; 269 } 270 271 static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, 272 struct hns_roce_mr *mr) 273 { 274 struct device *dev = &hr_dev->pdev->dev; 275 int npages = 0; 276 int ret; 277 278 if (mr->enabled) { 279 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key) 280 & (hr_dev->caps.num_mtpts - 1)); 281 if (ret) 282 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret); 283 } 284 285 if (mr->size != ~0ULL) { 286 npages = ib_umem_page_count(mr->umem); 287 dma_free_coherent(dev, (unsigned int)(npages * 8), mr->pbl_buf, 288 mr->pbl_dma_addr); 289 } 290 291 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, 292 key_to_hw_index(mr->key), BITMAP_NO_RR); 293 } 294 295 static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, 296 struct hns_roce_mr *mr) 297 { 298 int ret; 299 unsigned long mtpt_idx = key_to_hw_index(mr->key); 300 struct device *dev = &hr_dev->pdev->dev; 301 struct hns_roce_cmd_mailbox *mailbox; 302 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 303 304 /* Prepare HEM entry memory */ 305 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx); 306 if (ret) 307 return ret; 308 309 /* Allocate mailbox memory */ 310 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 311 if (IS_ERR(mailbox)) { 312 ret = PTR_ERR(mailbox); 313 goto err_table; 314 } 315 316 ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx); 317 if (ret) { 318 dev_err(dev, "Write mtpt fail!\n"); 319 goto err_page; 320 } 321 322 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, 323 mtpt_idx & (hr_dev->caps.num_mtpts - 1)); 324 if (ret) { 325 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret); 326 goto err_page; 327 } 328 329 mr->enabled = 1; 330 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 331 332 return 0; 333 334 err_page: 335 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 336 337 err_table: 338 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx); 339 return ret; 340 } 341 342 static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev, 343 struct hns_roce_mtt *mtt, u32 start_index, 344 u32 npages, u64 *page_list) 345 { 346 u32 i = 0; 347 __le64 *mtts = NULL; 348 dma_addr_t dma_handle; 349 u32 s = start_index * sizeof(u64); 350 351 /* All MTTs must fit in the same page */ 352 if (start_index / (PAGE_SIZE / sizeof(u64)) != 353 (start_index + npages - 1) / (PAGE_SIZE / sizeof(u64))) 354 return -EINVAL; 355 356 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1)) 357 return -EINVAL; 358 359 mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table, 360 mtt->first_seg + s / hr_dev->caps.mtt_entry_sz, 361 &dma_handle); 362 if (!mtts) 363 return -ENOMEM; 364 365 /* Save page addr, low 12 bits : 0 */ 366 for (i = 0; i < npages; ++i) 367 mtts[i] = (cpu_to_le64(page_list[i])) >> PAGE_ADDR_SHIFT; 368 369 return 0; 370 } 371 372 static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev, 373 struct hns_roce_mtt *mtt, u32 start_index, 374 u32 npages, u64 *page_list) 375 { 376 int chunk; 377 int ret; 378 379 if (mtt->order < 0) 380 return -EINVAL; 381 382 while (npages > 0) { 383 chunk = min_t(int, PAGE_SIZE / sizeof(u64), npages); 384 385 ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk, 386 page_list); 387 if (ret) 388 return ret; 389 390 npages -= chunk; 391 start_index += chunk; 392 page_list += chunk; 393 } 394 395 return 0; 396 } 397 398 int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, 399 struct hns_roce_mtt *mtt, struct hns_roce_buf *buf) 400 { 401 u32 i = 0; 402 int ret = 0; 403 u64 *page_list = NULL; 404 405 page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL); 406 if (!page_list) 407 return -ENOMEM; 408 409 for (i = 0; i < buf->npages; ++i) { 410 if (buf->nbufs == 1) 411 page_list[i] = buf->direct.map + (i << buf->page_shift); 412 else 413 page_list[i] = buf->page_list[i].map; 414 415 } 416 ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list); 417 418 kfree(page_list); 419 420 return ret; 421 } 422 423 int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev) 424 { 425 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 426 int ret = 0; 427 428 ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap, 429 hr_dev->caps.num_mtpts, 430 hr_dev->caps.num_mtpts - 1, 431 hr_dev->caps.reserved_mrws, 0); 432 if (ret) 433 return ret; 434 435 ret = hns_roce_buddy_init(&mr_table->mtt_buddy, 436 ilog2(hr_dev->caps.num_mtt_segs)); 437 if (ret) 438 goto err_buddy; 439 440 return 0; 441 442 err_buddy: 443 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); 444 return ret; 445 } 446 447 void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev) 448 { 449 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table; 450 451 hns_roce_buddy_cleanup(&mr_table->mtt_buddy); 452 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap); 453 } 454 455 struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc) 456 { 457 int ret = 0; 458 struct hns_roce_mr *mr = NULL; 459 460 mr = kmalloc(sizeof(*mr), GFP_KERNEL); 461 if (mr == NULL) 462 return ERR_PTR(-ENOMEM); 463 464 /* Allocate memory region key */ 465 ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0, 466 ~0ULL, acc, 0, mr); 467 if (ret) 468 goto err_free; 469 470 ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr); 471 if (ret) 472 goto err_mr; 473 474 mr->ibmr.rkey = mr->ibmr.lkey = mr->key; 475 mr->umem = NULL; 476 477 return &mr->ibmr; 478 479 err_mr: 480 hns_roce_mr_free(to_hr_dev(pd->device), mr); 481 482 err_free: 483 kfree(mr); 484 return ERR_PTR(ret); 485 } 486 487 int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, 488 struct hns_roce_mtt *mtt, struct ib_umem *umem) 489 { 490 struct scatterlist *sg; 491 int i, k, entry; 492 int ret = 0; 493 u64 *pages; 494 u32 n; 495 int len; 496 497 pages = (u64 *) __get_free_page(GFP_KERNEL); 498 if (!pages) 499 return -ENOMEM; 500 501 i = n = 0; 502 503 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 504 len = sg_dma_len(sg) >> mtt->page_shift; 505 for (k = 0; k < len; ++k) { 506 pages[i++] = sg_dma_address(sg) + umem->page_size * k; 507 if (i == PAGE_SIZE / sizeof(u64)) { 508 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, 509 pages); 510 if (ret) 511 goto out; 512 n += i; 513 i = 0; 514 } 515 } 516 } 517 518 if (i) 519 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages); 520 521 out: 522 free_page((unsigned long) pages); 523 return ret; 524 } 525 526 static int hns_roce_ib_umem_write_mr(struct hns_roce_mr *mr, 527 struct ib_umem *umem) 528 { 529 int i = 0; 530 int entry; 531 struct scatterlist *sg; 532 533 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 534 mr->pbl_buf[i] = ((u64)sg_dma_address(sg)) >> 12; 535 i++; 536 } 537 538 /* Memory barrier */ 539 mb(); 540 541 return 0; 542 } 543 544 struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 545 u64 virt_addr, int access_flags, 546 struct ib_udata *udata) 547 { 548 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 549 struct device *dev = &hr_dev->pdev->dev; 550 struct hns_roce_mr *mr = NULL; 551 int ret = 0; 552 int n = 0; 553 554 mr = kmalloc(sizeof(*mr), GFP_KERNEL); 555 if (!mr) 556 return ERR_PTR(-ENOMEM); 557 558 mr->umem = ib_umem_get(pd->uobject->context, start, length, 559 access_flags, 0); 560 if (IS_ERR(mr->umem)) { 561 ret = PTR_ERR(mr->umem); 562 goto err_free; 563 } 564 565 n = ib_umem_page_count(mr->umem); 566 if (mr->umem->page_size != HNS_ROCE_HEM_PAGE_SIZE) { 567 dev_err(dev, "Just support 4K page size but is 0x%x now!\n", 568 mr->umem->page_size); 569 ret = -EINVAL; 570 goto err_umem; 571 } 572 573 if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) { 574 dev_err(dev, " MR len %lld err. MR is limited to 4G at most!\n", 575 length); 576 ret = -EINVAL; 577 goto err_umem; 578 } 579 580 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length, 581 access_flags, n, mr); 582 if (ret) 583 goto err_umem; 584 585 ret = hns_roce_ib_umem_write_mr(mr, mr->umem); 586 if (ret) 587 goto err_mr; 588 589 ret = hns_roce_mr_enable(hr_dev, mr); 590 if (ret) 591 goto err_mr; 592 593 mr->ibmr.rkey = mr->ibmr.lkey = mr->key; 594 595 return &mr->ibmr; 596 597 err_mr: 598 hns_roce_mr_free(hr_dev, mr); 599 600 err_umem: 601 ib_umem_release(mr->umem); 602 603 err_free: 604 kfree(mr); 605 return ERR_PTR(ret); 606 } 607 608 int hns_roce_dereg_mr(struct ib_mr *ibmr) 609 { 610 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device); 611 struct hns_roce_mr *mr = to_hr_mr(ibmr); 612 int ret = 0; 613 614 if (hr_dev->hw->dereg_mr) { 615 ret = hr_dev->hw->dereg_mr(hr_dev, mr); 616 } else { 617 hns_roce_mr_free(hr_dev, mr); 618 619 if (mr->umem) 620 ib_umem_release(mr->umem); 621 622 kfree(mr); 623 } 624 625 return ret; 626 } 627