1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <linux/init.h> 36 #include <linux/errno.h> 37 #include <linux/export.h> 38 #include <linux/slab.h> 39 #include <linux/kernel.h> 40 #include <linux/vmalloc.h> 41 42 #include <linux/mlx4/cmd.h> 43 44 #include "mlx4.h" 45 #include "icm.h" 46 47 static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) 48 { 49 int o; 50 int m; 51 u32 seg; 52 53 spin_lock(&buddy->lock); 54 55 for (o = order; o <= buddy->max_order; ++o) 56 if (buddy->num_free[o]) { 57 m = 1 << (buddy->max_order - o); 58 seg = find_first_bit(buddy->bits[o], m); 59 if (seg < m) 60 goto found; 61 } 62 63 spin_unlock(&buddy->lock); 64 return -1; 65 66 found: 67 clear_bit(seg, buddy->bits[o]); 68 --buddy->num_free[o]; 69 70 while (o > order) { 71 --o; 72 seg <<= 1; 73 set_bit(seg ^ 1, buddy->bits[o]); 74 ++buddy->num_free[o]; 75 } 76 77 spin_unlock(&buddy->lock); 78 79 seg <<= order; 80 81 return seg; 82 } 83 84 static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) 85 { 86 seg >>= order; 87 88 spin_lock(&buddy->lock); 89 90 while (test_bit(seg ^ 1, buddy->bits[order])) { 91 clear_bit(seg ^ 1, buddy->bits[order]); 92 --buddy->num_free[order]; 93 seg >>= 1; 94 ++order; 95 } 96 97 set_bit(seg, buddy->bits[order]); 98 ++buddy->num_free[order]; 99 100 spin_unlock(&buddy->lock); 101 } 102 103 static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) 104 { 105 int i, s; 106 107 buddy->max_order = max_order; 108 spin_lock_init(&buddy->lock); 109 110 buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *), 111 GFP_KERNEL); 112 buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, 113 GFP_KERNEL); 114 if (!buddy->bits || !buddy->num_free) 115 goto err_out; 116 117 for (i = 0; i <= buddy->max_order; ++i) { 118 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 119 buddy->bits[i] = kcalloc(s, sizeof (long), GFP_KERNEL | __GFP_NOWARN); 120 if (!buddy->bits[i]) { 121 buddy->bits[i] = vzalloc(s * sizeof(long)); 122 if (!buddy->bits[i]) 123 goto err_out_free; 124 } 125 } 126 127 set_bit(0, buddy->bits[buddy->max_order]); 128 buddy->num_free[buddy->max_order] = 1; 129 130 return 0; 131 132 err_out_free: 133 for (i = 0; i <= buddy->max_order; ++i) 134 if (buddy->bits[i] && is_vmalloc_addr(buddy->bits[i])) 135 vfree(buddy->bits[i]); 136 else 137 kfree(buddy->bits[i]); 138 139 err_out: 140 kfree(buddy->bits); 141 kfree(buddy->num_free); 142 143 return -ENOMEM; 144 } 145 146 static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) 147 { 148 int i; 149 150 for (i = 0; i <= buddy->max_order; ++i) 151 if (is_vmalloc_addr(buddy->bits[i])) 152 vfree(buddy->bits[i]); 153 else 154 kfree(buddy->bits[i]); 155 156 kfree(buddy->bits); 157 kfree(buddy->num_free); 158 } 159 160 u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 161 { 162 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 163 u32 seg; 164 int seg_order; 165 u32 offset; 166 167 seg_order = max_t(int, order - log_mtts_per_seg, 0); 168 169 seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, seg_order); 170 if (seg == -1) 171 return -1; 172 173 offset = seg * (1 << log_mtts_per_seg); 174 175 if (mlx4_table_get_range(dev, &mr_table->mtt_table, offset, 176 offset + (1 << order) - 1)) { 177 mlx4_buddy_free(&mr_table->mtt_buddy, seg, seg_order); 178 return -1; 179 } 180 181 return offset; 182 } 183 184 static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 185 { 186 u64 in_param = 0; 187 u64 out_param; 188 int err; 189 190 if (mlx4_is_mfunc(dev)) { 191 set_param_l(&in_param, order); 192 err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT, 193 RES_OP_RESERVE_AND_MAP, 194 MLX4_CMD_ALLOC_RES, 195 MLX4_CMD_TIME_CLASS_A, 196 MLX4_CMD_WRAPPED); 197 if (err) 198 return -1; 199 return get_param_l(&out_param); 200 } 201 return __mlx4_alloc_mtt_range(dev, order); 202 } 203 204 int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, 205 struct mlx4_mtt *mtt) 206 { 207 int i; 208 209 if (!npages) { 210 mtt->order = -1; 211 mtt->page_shift = MLX4_ICM_PAGE_SHIFT; 212 return 0; 213 } else 214 mtt->page_shift = page_shift; 215 216 for (mtt->order = 0, i = 1; i < npages; i <<= 1) 217 ++mtt->order; 218 219 mtt->offset = mlx4_alloc_mtt_range(dev, mtt->order); 220 if (mtt->offset == -1) 221 return -ENOMEM; 222 223 return 0; 224 } 225 EXPORT_SYMBOL_GPL(mlx4_mtt_init); 226 227 void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 228 { 229 u32 first_seg; 230 int seg_order; 231 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 232 233 seg_order = max_t(int, order - log_mtts_per_seg, 0); 234 first_seg = offset / (1 << log_mtts_per_seg); 235 236 mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, seg_order); 237 mlx4_table_put_range(dev, &mr_table->mtt_table, offset, 238 offset + (1 << order) - 1); 239 } 240 241 static void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 242 { 243 u64 in_param = 0; 244 int err; 245 246 if (mlx4_is_mfunc(dev)) { 247 set_param_l(&in_param, offset); 248 set_param_h(&in_param, order); 249 err = mlx4_cmd(dev, in_param, RES_MTT, RES_OP_RESERVE_AND_MAP, 250 MLX4_CMD_FREE_RES, 251 MLX4_CMD_TIME_CLASS_A, 252 MLX4_CMD_WRAPPED); 253 if (err) 254 mlx4_warn(dev, "Failed to free mtt range at:" 255 "%d order:%d\n", offset, order); 256 return; 257 } 258 __mlx4_free_mtt_range(dev, offset, order); 259 } 260 261 void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 262 { 263 if (mtt->order < 0) 264 return; 265 266 mlx4_free_mtt_range(dev, mtt->offset, mtt->order); 267 } 268 EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); 269 270 u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 271 { 272 return (u64) mtt->offset * dev->caps.mtt_entry_sz; 273 } 274 EXPORT_SYMBOL_GPL(mlx4_mtt_addr); 275 276 static u32 hw_index_to_key(u32 ind) 277 { 278 return (ind >> 24) | (ind << 8); 279 } 280 281 static u32 key_to_hw_index(u32 key) 282 { 283 return (key << 24) | (key >> 8); 284 } 285 286 static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 287 int mpt_index) 288 { 289 return mlx4_cmd(dev, mailbox->dma, mpt_index, 290 0, MLX4_CMD_SW2HW_MPT, MLX4_CMD_TIME_CLASS_B, 291 MLX4_CMD_WRAPPED); 292 } 293 294 static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 295 int mpt_index) 296 { 297 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, 298 !mailbox, MLX4_CMD_HW2SW_MPT, 299 MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); 300 } 301 302 static int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, 303 u64 iova, u64 size, u32 access, int npages, 304 int page_shift, struct mlx4_mr *mr) 305 { 306 mr->iova = iova; 307 mr->size = size; 308 mr->pd = pd; 309 mr->access = access; 310 mr->enabled = MLX4_MPT_DISABLED; 311 mr->key = hw_index_to_key(mridx); 312 313 return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); 314 } 315 316 static int mlx4_WRITE_MTT(struct mlx4_dev *dev, 317 struct mlx4_cmd_mailbox *mailbox, 318 int num_entries) 319 { 320 return mlx4_cmd(dev, mailbox->dma, num_entries, 0, MLX4_CMD_WRITE_MTT, 321 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 322 } 323 324 int __mlx4_mpt_reserve(struct mlx4_dev *dev) 325 { 326 struct mlx4_priv *priv = mlx4_priv(dev); 327 328 return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); 329 } 330 331 static int mlx4_mpt_reserve(struct mlx4_dev *dev) 332 { 333 u64 out_param; 334 335 if (mlx4_is_mfunc(dev)) { 336 if (mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, RES_OP_RESERVE, 337 MLX4_CMD_ALLOC_RES, 338 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 339 return -1; 340 return get_param_l(&out_param); 341 } 342 return __mlx4_mpt_reserve(dev); 343 } 344 345 void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index) 346 { 347 struct mlx4_priv *priv = mlx4_priv(dev); 348 349 mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index); 350 } 351 352 static void mlx4_mpt_release(struct mlx4_dev *dev, u32 index) 353 { 354 u64 in_param = 0; 355 356 if (mlx4_is_mfunc(dev)) { 357 set_param_l(&in_param, index); 358 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_RESERVE, 359 MLX4_CMD_FREE_RES, 360 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 361 mlx4_warn(dev, "Failed to release mr index:%d\n", 362 index); 363 return; 364 } 365 __mlx4_mpt_release(dev, index); 366 } 367 368 int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) 369 { 370 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 371 372 return mlx4_table_get(dev, &mr_table->dmpt_table, index); 373 } 374 375 static int mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) 376 { 377 u64 param = 0; 378 379 if (mlx4_is_mfunc(dev)) { 380 set_param_l(¶m, index); 381 return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, RES_OP_MAP_ICM, 382 MLX4_CMD_ALLOC_RES, 383 MLX4_CMD_TIME_CLASS_A, 384 MLX4_CMD_WRAPPED); 385 } 386 return __mlx4_mpt_alloc_icm(dev, index); 387 } 388 389 void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) 390 { 391 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 392 393 mlx4_table_put(dev, &mr_table->dmpt_table, index); 394 } 395 396 static void mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) 397 { 398 u64 in_param = 0; 399 400 if (mlx4_is_mfunc(dev)) { 401 set_param_l(&in_param, index); 402 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_MAP_ICM, 403 MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, 404 MLX4_CMD_WRAPPED)) 405 mlx4_warn(dev, "Failed to free icm of mr index:%d\n", 406 index); 407 return; 408 } 409 return __mlx4_mpt_free_icm(dev, index); 410 } 411 412 int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, 413 int npages, int page_shift, struct mlx4_mr *mr) 414 { 415 u32 index; 416 int err; 417 418 index = mlx4_mpt_reserve(dev); 419 if (index == -1) 420 return -ENOMEM; 421 422 err = mlx4_mr_alloc_reserved(dev, index, pd, iova, size, 423 access, npages, page_shift, mr); 424 if (err) 425 mlx4_mpt_release(dev, index); 426 427 return err; 428 } 429 EXPORT_SYMBOL_GPL(mlx4_mr_alloc); 430 431 static int mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) 432 { 433 int err; 434 435 if (mr->enabled == MLX4_MPT_EN_HW) { 436 err = mlx4_HW2SW_MPT(dev, NULL, 437 key_to_hw_index(mr->key) & 438 (dev->caps.num_mpts - 1)); 439 if (err) { 440 mlx4_warn(dev, "HW2SW_MPT failed (%d),", err); 441 mlx4_warn(dev, "MR has MWs bound to it.\n"); 442 return err; 443 } 444 445 mr->enabled = MLX4_MPT_EN_SW; 446 } 447 mlx4_mtt_cleanup(dev, &mr->mtt); 448 449 return 0; 450 } 451 452 int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) 453 { 454 int ret; 455 456 ret = mlx4_mr_free_reserved(dev, mr); 457 if (ret) 458 return ret; 459 if (mr->enabled) 460 mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); 461 mlx4_mpt_release(dev, key_to_hw_index(mr->key)); 462 463 return 0; 464 } 465 EXPORT_SYMBOL_GPL(mlx4_mr_free); 466 467 int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) 468 { 469 struct mlx4_cmd_mailbox *mailbox; 470 struct mlx4_mpt_entry *mpt_entry; 471 int err; 472 473 err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mr->key)); 474 if (err) 475 return err; 476 477 mailbox = mlx4_alloc_cmd_mailbox(dev); 478 if (IS_ERR(mailbox)) { 479 err = PTR_ERR(mailbox); 480 goto err_table; 481 } 482 mpt_entry = mailbox->buf; 483 484 memset(mpt_entry, 0, sizeof *mpt_entry); 485 486 mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_MIO | 487 MLX4_MPT_FLAG_REGION | 488 mr->access); 489 490 mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); 491 mpt_entry->pd_flags = cpu_to_be32(mr->pd | MLX4_MPT_PD_FLAG_EN_INV); 492 mpt_entry->start = cpu_to_be64(mr->iova); 493 mpt_entry->length = cpu_to_be64(mr->size); 494 mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); 495 496 if (mr->mtt.order < 0) { 497 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); 498 mpt_entry->mtt_addr = 0; 499 } else { 500 mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, 501 &mr->mtt)); 502 } 503 504 if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { 505 /* fast register MR in free state */ 506 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); 507 mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | 508 MLX4_MPT_PD_FLAG_RAE); 509 mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); 510 } else { 511 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); 512 } 513 514 err = mlx4_SW2HW_MPT(dev, mailbox, 515 key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); 516 if (err) { 517 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 518 goto err_cmd; 519 } 520 mr->enabled = MLX4_MPT_EN_HW; 521 522 mlx4_free_cmd_mailbox(dev, mailbox); 523 524 return 0; 525 526 err_cmd: 527 mlx4_free_cmd_mailbox(dev, mailbox); 528 529 err_table: 530 mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); 531 return err; 532 } 533 EXPORT_SYMBOL_GPL(mlx4_mr_enable); 534 535 static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 536 int start_index, int npages, u64 *page_list) 537 { 538 struct mlx4_priv *priv = mlx4_priv(dev); 539 __be64 *mtts; 540 dma_addr_t dma_handle; 541 int i; 542 543 mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->offset + 544 start_index, &dma_handle); 545 546 if (!mtts) 547 return -ENOMEM; 548 549 dma_sync_single_for_cpu(&dev->pdev->dev, dma_handle, 550 npages * sizeof (u64), DMA_TO_DEVICE); 551 552 for (i = 0; i < npages; ++i) 553 mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 554 555 dma_sync_single_for_device(&dev->pdev->dev, dma_handle, 556 npages * sizeof (u64), DMA_TO_DEVICE); 557 558 return 0; 559 } 560 561 int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 562 int start_index, int npages, u64 *page_list) 563 { 564 int err = 0; 565 int chunk; 566 int mtts_per_page; 567 int max_mtts_first_page; 568 569 /* compute how may mtts fit in the first page */ 570 mtts_per_page = PAGE_SIZE / sizeof(u64); 571 max_mtts_first_page = mtts_per_page - (mtt->offset + start_index) 572 % mtts_per_page; 573 574 chunk = min_t(int, max_mtts_first_page, npages); 575 576 while (npages > 0) { 577 err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list); 578 if (err) 579 return err; 580 npages -= chunk; 581 start_index += chunk; 582 page_list += chunk; 583 584 chunk = min_t(int, mtts_per_page, npages); 585 } 586 return err; 587 } 588 589 int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 590 int start_index, int npages, u64 *page_list) 591 { 592 struct mlx4_cmd_mailbox *mailbox = NULL; 593 __be64 *inbox = NULL; 594 int chunk; 595 int err = 0; 596 int i; 597 598 if (mtt->order < 0) 599 return -EINVAL; 600 601 if (mlx4_is_mfunc(dev)) { 602 mailbox = mlx4_alloc_cmd_mailbox(dev); 603 if (IS_ERR(mailbox)) 604 return PTR_ERR(mailbox); 605 inbox = mailbox->buf; 606 607 while (npages > 0) { 608 chunk = min_t(int, MLX4_MAILBOX_SIZE / sizeof(u64) - 2, 609 npages); 610 inbox[0] = cpu_to_be64(mtt->offset + start_index); 611 inbox[1] = 0; 612 for (i = 0; i < chunk; ++i) 613 inbox[i + 2] = cpu_to_be64(page_list[i] | 614 MLX4_MTT_FLAG_PRESENT); 615 err = mlx4_WRITE_MTT(dev, mailbox, chunk); 616 if (err) { 617 mlx4_free_cmd_mailbox(dev, mailbox); 618 return err; 619 } 620 621 npages -= chunk; 622 start_index += chunk; 623 page_list += chunk; 624 } 625 mlx4_free_cmd_mailbox(dev, mailbox); 626 return err; 627 } 628 629 return __mlx4_write_mtt(dev, mtt, start_index, npages, page_list); 630 } 631 EXPORT_SYMBOL_GPL(mlx4_write_mtt); 632 633 int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 634 struct mlx4_buf *buf) 635 { 636 u64 *page_list; 637 int err; 638 int i; 639 640 page_list = kmalloc(buf->npages * sizeof *page_list, GFP_KERNEL); 641 if (!page_list) 642 return -ENOMEM; 643 644 for (i = 0; i < buf->npages; ++i) 645 if (buf->nbufs == 1) 646 page_list[i] = buf->direct.map + (i << buf->page_shift); 647 else 648 page_list[i] = buf->page_list[i].map; 649 650 err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); 651 652 kfree(page_list); 653 return err; 654 } 655 EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); 656 657 int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, 658 struct mlx4_mw *mw) 659 { 660 u32 index; 661 662 if ((type == MLX4_MW_TYPE_1 && 663 !(dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW)) || 664 (type == MLX4_MW_TYPE_2 && 665 !(dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN))) 666 return -ENOTSUPP; 667 668 index = mlx4_mpt_reserve(dev); 669 if (index == -1) 670 return -ENOMEM; 671 672 mw->key = hw_index_to_key(index); 673 mw->pd = pd; 674 mw->type = type; 675 mw->enabled = MLX4_MPT_DISABLED; 676 677 return 0; 678 } 679 EXPORT_SYMBOL_GPL(mlx4_mw_alloc); 680 681 int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw) 682 { 683 struct mlx4_cmd_mailbox *mailbox; 684 struct mlx4_mpt_entry *mpt_entry; 685 int err; 686 687 err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mw->key)); 688 if (err) 689 return err; 690 691 mailbox = mlx4_alloc_cmd_mailbox(dev); 692 if (IS_ERR(mailbox)) { 693 err = PTR_ERR(mailbox); 694 goto err_table; 695 } 696 mpt_entry = mailbox->buf; 697 698 memset(mpt_entry, 0, sizeof(*mpt_entry)); 699 700 /* Note that the MLX4_MPT_FLAG_REGION bit in mpt_entry->flags is turned 701 * off, thus creating a memory window and not a memory region. 702 */ 703 mpt_entry->key = cpu_to_be32(key_to_hw_index(mw->key)); 704 mpt_entry->pd_flags = cpu_to_be32(mw->pd); 705 if (mw->type == MLX4_MW_TYPE_2) { 706 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); 707 mpt_entry->qpn = cpu_to_be32(MLX4_MPT_QP_FLAG_BOUND_QP); 708 mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_EN_INV); 709 } 710 711 err = mlx4_SW2HW_MPT(dev, mailbox, 712 key_to_hw_index(mw->key) & 713 (dev->caps.num_mpts - 1)); 714 if (err) { 715 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 716 goto err_cmd; 717 } 718 mw->enabled = MLX4_MPT_EN_HW; 719 720 mlx4_free_cmd_mailbox(dev, mailbox); 721 722 return 0; 723 724 err_cmd: 725 mlx4_free_cmd_mailbox(dev, mailbox); 726 727 err_table: 728 mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); 729 return err; 730 } 731 EXPORT_SYMBOL_GPL(mlx4_mw_enable); 732 733 void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw) 734 { 735 int err; 736 737 if (mw->enabled == MLX4_MPT_EN_HW) { 738 err = mlx4_HW2SW_MPT(dev, NULL, 739 key_to_hw_index(mw->key) & 740 (dev->caps.num_mpts - 1)); 741 if (err) 742 mlx4_warn(dev, "xxx HW2SW_MPT failed (%d)\n", err); 743 744 mw->enabled = MLX4_MPT_EN_SW; 745 } 746 if (mw->enabled) 747 mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); 748 mlx4_mpt_release(dev, key_to_hw_index(mw->key)); 749 } 750 EXPORT_SYMBOL_GPL(mlx4_mw_free); 751 752 int mlx4_init_mr_table(struct mlx4_dev *dev) 753 { 754 struct mlx4_priv *priv = mlx4_priv(dev); 755 struct mlx4_mr_table *mr_table = &priv->mr_table; 756 int err; 757 758 if (!is_power_of_2(dev->caps.num_mpts)) 759 return -EINVAL; 760 761 /* Nothing to do for slaves - all MR handling is forwarded 762 * to the master */ 763 if (mlx4_is_slave(dev)) 764 return 0; 765 766 err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, 767 ~0, dev->caps.reserved_mrws, 0); 768 if (err) 769 return err; 770 771 err = mlx4_buddy_init(&mr_table->mtt_buddy, 772 ilog2((u32)dev->caps.num_mtts / 773 (1 << log_mtts_per_seg))); 774 if (err) 775 goto err_buddy; 776 777 if (dev->caps.reserved_mtts) { 778 priv->reserved_mtts = 779 mlx4_alloc_mtt_range(dev, 780 fls(dev->caps.reserved_mtts - 1)); 781 if (priv->reserved_mtts < 0) { 782 mlx4_warn(dev, "MTT table of order %u is too small.\n", 783 mr_table->mtt_buddy.max_order); 784 err = -ENOMEM; 785 goto err_reserve_mtts; 786 } 787 } 788 789 return 0; 790 791 err_reserve_mtts: 792 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 793 794 err_buddy: 795 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 796 797 return err; 798 } 799 800 void mlx4_cleanup_mr_table(struct mlx4_dev *dev) 801 { 802 struct mlx4_priv *priv = mlx4_priv(dev); 803 struct mlx4_mr_table *mr_table = &priv->mr_table; 804 805 if (mlx4_is_slave(dev)) 806 return; 807 if (priv->reserved_mtts >= 0) 808 mlx4_free_mtt_range(dev, priv->reserved_mtts, 809 fls(dev->caps.reserved_mtts - 1)); 810 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 811 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 812 } 813 814 static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list, 815 int npages, u64 iova) 816 { 817 int i, page_mask; 818 819 if (npages > fmr->max_pages) 820 return -EINVAL; 821 822 page_mask = (1 << fmr->page_shift) - 1; 823 824 /* We are getting page lists, so va must be page aligned. */ 825 if (iova & page_mask) 826 return -EINVAL; 827 828 /* Trust the user not to pass misaligned data in page_list */ 829 if (0) 830 for (i = 0; i < npages; ++i) { 831 if (page_list[i] & ~page_mask) 832 return -EINVAL; 833 } 834 835 if (fmr->maps >= fmr->max_maps) 836 return -EINVAL; 837 838 return 0; 839 } 840 841 int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, 842 int npages, u64 iova, u32 *lkey, u32 *rkey) 843 { 844 u32 key; 845 int i, err; 846 847 err = mlx4_check_fmr(fmr, page_list, npages, iova); 848 if (err) 849 return err; 850 851 ++fmr->maps; 852 853 key = key_to_hw_index(fmr->mr.key); 854 key += dev->caps.num_mpts; 855 *lkey = *rkey = fmr->mr.key = hw_index_to_key(key); 856 857 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; 858 859 /* Make sure MPT status is visible before writing MTT entries */ 860 wmb(); 861 862 dma_sync_single_for_cpu(&dev->pdev->dev, fmr->dma_handle, 863 npages * sizeof(u64), DMA_TO_DEVICE); 864 865 for (i = 0; i < npages; ++i) 866 fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 867 868 dma_sync_single_for_device(&dev->pdev->dev, fmr->dma_handle, 869 npages * sizeof(u64), DMA_TO_DEVICE); 870 871 fmr->mpt->key = cpu_to_be32(key); 872 fmr->mpt->lkey = cpu_to_be32(key); 873 fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift)); 874 fmr->mpt->start = cpu_to_be64(iova); 875 876 /* Make MTT entries are visible before setting MPT status */ 877 wmb(); 878 879 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW; 880 881 /* Make sure MPT status is visible before consumer can use FMR */ 882 wmb(); 883 884 return 0; 885 } 886 EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr); 887 888 int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, 889 int max_maps, u8 page_shift, struct mlx4_fmr *fmr) 890 { 891 struct mlx4_priv *priv = mlx4_priv(dev); 892 int err = -ENOMEM; 893 894 if (max_maps > dev->caps.max_fmr_maps) 895 return -EINVAL; 896 897 if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) 898 return -EINVAL; 899 900 /* All MTTs must fit in the same page */ 901 if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) 902 return -EINVAL; 903 904 fmr->page_shift = page_shift; 905 fmr->max_pages = max_pages; 906 fmr->max_maps = max_maps; 907 fmr->maps = 0; 908 909 err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages, 910 page_shift, &fmr->mr); 911 if (err) 912 return err; 913 914 fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, 915 fmr->mr.mtt.offset, 916 &fmr->dma_handle); 917 918 if (!fmr->mtts) { 919 err = -ENOMEM; 920 goto err_free; 921 } 922 923 return 0; 924 925 err_free: 926 (void) mlx4_mr_free(dev, &fmr->mr); 927 return err; 928 } 929 EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); 930 931 int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 932 { 933 struct mlx4_priv *priv = mlx4_priv(dev); 934 int err; 935 936 err = mlx4_mr_enable(dev, &fmr->mr); 937 if (err) 938 return err; 939 940 fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, 941 key_to_hw_index(fmr->mr.key), NULL); 942 if (!fmr->mpt) 943 return -ENOMEM; 944 945 return 0; 946 } 947 EXPORT_SYMBOL_GPL(mlx4_fmr_enable); 948 949 void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, 950 u32 *lkey, u32 *rkey) 951 { 952 struct mlx4_cmd_mailbox *mailbox; 953 int err; 954 955 if (!fmr->maps) 956 return; 957 958 fmr->maps = 0; 959 960 mailbox = mlx4_alloc_cmd_mailbox(dev); 961 if (IS_ERR(mailbox)) { 962 err = PTR_ERR(mailbox); 963 printk(KERN_WARNING "mlx4_ib: mlx4_alloc_cmd_mailbox" 964 " failed (%d)\n", err); 965 return; 966 } 967 968 err = mlx4_HW2SW_MPT(dev, NULL, 969 key_to_hw_index(fmr->mr.key) & 970 (dev->caps.num_mpts - 1)); 971 mlx4_free_cmd_mailbox(dev, mailbox); 972 if (err) { 973 printk(KERN_WARNING "mlx4_ib: mlx4_HW2SW_MPT failed (%d)\n", 974 err); 975 return; 976 } 977 fmr->mr.enabled = MLX4_MPT_EN_SW; 978 } 979 EXPORT_SYMBOL_GPL(mlx4_fmr_unmap); 980 981 int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 982 { 983 int ret; 984 985 if (fmr->maps) 986 return -EBUSY; 987 988 ret = mlx4_mr_free(dev, &fmr->mr); 989 if (ret) 990 return ret; 991 fmr->mr.enabled = MLX4_MPT_DISABLED; 992 993 return 0; 994 } 995 EXPORT_SYMBOL_GPL(mlx4_fmr_free); 996 997 int mlx4_SYNC_TPT(struct mlx4_dev *dev) 998 { 999 return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000, 1000 MLX4_CMD_NATIVE); 1001 } 1002 EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT); 1003