1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <linux/init.h> 36 #include <linux/errno.h> 37 #include <linux/export.h> 38 #include <linux/slab.h> 39 #include <linux/kernel.h> 40 #include <linux/vmalloc.h> 41 42 #include <linux/mlx4/cmd.h> 43 44 #include "mlx4.h" 45 #include "icm.h" 46 47 #define MLX4_MPT_FLAG_SW_OWNS (0xfUL << 28) 48 #define MLX4_MPT_FLAG_FREE (0x3UL << 28) 49 #define MLX4_MPT_FLAG_MIO (1 << 17) 50 #define MLX4_MPT_FLAG_BIND_ENABLE (1 << 15) 51 #define MLX4_MPT_FLAG_PHYSICAL (1 << 9) 52 #define MLX4_MPT_FLAG_REGION (1 << 8) 53 54 #define MLX4_MPT_PD_FLAG_FAST_REG (1 << 27) 55 #define MLX4_MPT_PD_FLAG_RAE (1 << 28) 56 #define MLX4_MPT_PD_FLAG_EN_INV (3 << 24) 57 58 #define MLX4_MPT_STATUS_SW 0xF0 59 #define MLX4_MPT_STATUS_HW 0x00 60 61 static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) 62 { 63 int o; 64 int m; 65 u32 seg; 66 67 spin_lock(&buddy->lock); 68 69 for (o = order; o <= buddy->max_order; ++o) 70 if (buddy->num_free[o]) { 71 m = 1 << (buddy->max_order - o); 72 seg = find_first_bit(buddy->bits[o], m); 73 if (seg < m) 74 goto found; 75 } 76 77 spin_unlock(&buddy->lock); 78 return -1; 79 80 found: 81 clear_bit(seg, buddy->bits[o]); 82 --buddy->num_free[o]; 83 84 while (o > order) { 85 --o; 86 seg <<= 1; 87 set_bit(seg ^ 1, buddy->bits[o]); 88 ++buddy->num_free[o]; 89 } 90 91 spin_unlock(&buddy->lock); 92 93 seg <<= order; 94 95 return seg; 96 } 97 98 static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) 99 { 100 seg >>= order; 101 102 spin_lock(&buddy->lock); 103 104 while (test_bit(seg ^ 1, buddy->bits[order])) { 105 clear_bit(seg ^ 1, buddy->bits[order]); 106 --buddy->num_free[order]; 107 seg >>= 1; 108 ++order; 109 } 110 111 set_bit(seg, buddy->bits[order]); 112 ++buddy->num_free[order]; 113 114 spin_unlock(&buddy->lock); 115 } 116 117 static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) 118 { 119 int i, s; 120 121 buddy->max_order = max_order; 122 spin_lock_init(&buddy->lock); 123 124 buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *), 125 GFP_KERNEL); 126 buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, 127 GFP_KERNEL); 128 if (!buddy->bits || !buddy->num_free) 129 goto err_out; 130 131 for (i = 0; i <= buddy->max_order; ++i) { 132 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 133 buddy->bits[i] = kcalloc(s, sizeof (long), GFP_KERNEL | __GFP_NOWARN); 134 if (!buddy->bits[i]) { 135 buddy->bits[i] = vzalloc(s * sizeof(long)); 136 if (!buddy->bits[i]) 137 goto err_out_free; 138 } 139 } 140 141 set_bit(0, buddy->bits[buddy->max_order]); 142 buddy->num_free[buddy->max_order] = 1; 143 144 return 0; 145 146 err_out_free: 147 for (i = 0; i <= buddy->max_order; ++i) 148 if (buddy->bits[i] && is_vmalloc_addr(buddy->bits[i])) 149 vfree(buddy->bits[i]); 150 else 151 kfree(buddy->bits[i]); 152 153 err_out: 154 kfree(buddy->bits); 155 kfree(buddy->num_free); 156 157 return -ENOMEM; 158 } 159 160 static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) 161 { 162 int i; 163 164 for (i = 0; i <= buddy->max_order; ++i) 165 if (is_vmalloc_addr(buddy->bits[i])) 166 vfree(buddy->bits[i]); 167 else 168 kfree(buddy->bits[i]); 169 170 kfree(buddy->bits); 171 kfree(buddy->num_free); 172 } 173 174 u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 175 { 176 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 177 u32 seg; 178 int seg_order; 179 u32 offset; 180 181 seg_order = max_t(int, order - log_mtts_per_seg, 0); 182 183 seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, seg_order); 184 if (seg == -1) 185 return -1; 186 187 offset = seg * (1 << log_mtts_per_seg); 188 189 if (mlx4_table_get_range(dev, &mr_table->mtt_table, offset, 190 offset + (1 << order) - 1)) { 191 mlx4_buddy_free(&mr_table->mtt_buddy, seg, seg_order); 192 return -1; 193 } 194 195 return offset; 196 } 197 198 static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 199 { 200 u64 in_param; 201 u64 out_param; 202 int err; 203 204 if (mlx4_is_mfunc(dev)) { 205 set_param_l(&in_param, order); 206 err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT, 207 RES_OP_RESERVE_AND_MAP, 208 MLX4_CMD_ALLOC_RES, 209 MLX4_CMD_TIME_CLASS_A, 210 MLX4_CMD_WRAPPED); 211 if (err) 212 return -1; 213 return get_param_l(&out_param); 214 } 215 return __mlx4_alloc_mtt_range(dev, order); 216 } 217 218 int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, 219 struct mlx4_mtt *mtt) 220 { 221 int i; 222 223 if (!npages) { 224 mtt->order = -1; 225 mtt->page_shift = MLX4_ICM_PAGE_SHIFT; 226 return 0; 227 } else 228 mtt->page_shift = page_shift; 229 230 for (mtt->order = 0, i = 1; i < npages; i <<= 1) 231 ++mtt->order; 232 233 mtt->offset = mlx4_alloc_mtt_range(dev, mtt->order); 234 if (mtt->offset == -1) 235 return -ENOMEM; 236 237 return 0; 238 } 239 EXPORT_SYMBOL_GPL(mlx4_mtt_init); 240 241 void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 242 { 243 u32 first_seg; 244 int seg_order; 245 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 246 247 seg_order = max_t(int, order - log_mtts_per_seg, 0); 248 first_seg = offset / (1 << log_mtts_per_seg); 249 250 mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, seg_order); 251 mlx4_table_put_range(dev, &mr_table->mtt_table, offset, 252 offset + (1 << order) - 1); 253 } 254 255 static void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 256 { 257 u64 in_param; 258 int err; 259 260 if (mlx4_is_mfunc(dev)) { 261 set_param_l(&in_param, offset); 262 set_param_h(&in_param, order); 263 err = mlx4_cmd(dev, in_param, RES_MTT, RES_OP_RESERVE_AND_MAP, 264 MLX4_CMD_FREE_RES, 265 MLX4_CMD_TIME_CLASS_A, 266 MLX4_CMD_WRAPPED); 267 if (err) 268 mlx4_warn(dev, "Failed to free mtt range at:" 269 "%d order:%d\n", offset, order); 270 return; 271 } 272 __mlx4_free_mtt_range(dev, offset, order); 273 } 274 275 void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 276 { 277 if (mtt->order < 0) 278 return; 279 280 mlx4_free_mtt_range(dev, mtt->offset, mtt->order); 281 } 282 EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); 283 284 u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 285 { 286 return (u64) mtt->offset * dev->caps.mtt_entry_sz; 287 } 288 EXPORT_SYMBOL_GPL(mlx4_mtt_addr); 289 290 static u32 hw_index_to_key(u32 ind) 291 { 292 return (ind >> 24) | (ind << 8); 293 } 294 295 static u32 key_to_hw_index(u32 key) 296 { 297 return (key << 24) | (key >> 8); 298 } 299 300 static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 301 int mpt_index) 302 { 303 return mlx4_cmd(dev, mailbox->dma, mpt_index, 304 0, MLX4_CMD_SW2HW_MPT, MLX4_CMD_TIME_CLASS_B, 305 MLX4_CMD_WRAPPED); 306 } 307 308 static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 309 int mpt_index) 310 { 311 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, 312 !mailbox, MLX4_CMD_HW2SW_MPT, 313 MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); 314 } 315 316 static int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, 317 u64 iova, u64 size, u32 access, int npages, 318 int page_shift, struct mlx4_mr *mr) 319 { 320 mr->iova = iova; 321 mr->size = size; 322 mr->pd = pd; 323 mr->access = access; 324 mr->enabled = MLX4_MR_DISABLED; 325 mr->key = hw_index_to_key(mridx); 326 327 return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); 328 } 329 330 static int mlx4_WRITE_MTT(struct mlx4_dev *dev, 331 struct mlx4_cmd_mailbox *mailbox, 332 int num_entries) 333 { 334 return mlx4_cmd(dev, mailbox->dma, num_entries, 0, MLX4_CMD_WRITE_MTT, 335 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 336 } 337 338 int __mlx4_mr_reserve(struct mlx4_dev *dev) 339 { 340 struct mlx4_priv *priv = mlx4_priv(dev); 341 342 return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); 343 } 344 345 static int mlx4_mr_reserve(struct mlx4_dev *dev) 346 { 347 u64 out_param; 348 349 if (mlx4_is_mfunc(dev)) { 350 if (mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, RES_OP_RESERVE, 351 MLX4_CMD_ALLOC_RES, 352 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 353 return -1; 354 return get_param_l(&out_param); 355 } 356 return __mlx4_mr_reserve(dev); 357 } 358 359 void __mlx4_mr_release(struct mlx4_dev *dev, u32 index) 360 { 361 struct mlx4_priv *priv = mlx4_priv(dev); 362 363 mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index); 364 } 365 366 static void mlx4_mr_release(struct mlx4_dev *dev, u32 index) 367 { 368 u64 in_param; 369 370 if (mlx4_is_mfunc(dev)) { 371 set_param_l(&in_param, index); 372 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_RESERVE, 373 MLX4_CMD_FREE_RES, 374 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 375 mlx4_warn(dev, "Failed to release mr index:%d\n", 376 index); 377 return; 378 } 379 __mlx4_mr_release(dev, index); 380 } 381 382 int __mlx4_mr_alloc_icm(struct mlx4_dev *dev, u32 index) 383 { 384 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 385 386 return mlx4_table_get(dev, &mr_table->dmpt_table, index); 387 } 388 389 static int mlx4_mr_alloc_icm(struct mlx4_dev *dev, u32 index) 390 { 391 u64 param; 392 393 if (mlx4_is_mfunc(dev)) { 394 set_param_l(¶m, index); 395 return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, RES_OP_MAP_ICM, 396 MLX4_CMD_ALLOC_RES, 397 MLX4_CMD_TIME_CLASS_A, 398 MLX4_CMD_WRAPPED); 399 } 400 return __mlx4_mr_alloc_icm(dev, index); 401 } 402 403 void __mlx4_mr_free_icm(struct mlx4_dev *dev, u32 index) 404 { 405 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 406 407 mlx4_table_put(dev, &mr_table->dmpt_table, index); 408 } 409 410 static void mlx4_mr_free_icm(struct mlx4_dev *dev, u32 index) 411 { 412 u64 in_param; 413 414 if (mlx4_is_mfunc(dev)) { 415 set_param_l(&in_param, index); 416 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_MAP_ICM, 417 MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, 418 MLX4_CMD_WRAPPED)) 419 mlx4_warn(dev, "Failed to free icm of mr index:%d\n", 420 index); 421 return; 422 } 423 return __mlx4_mr_free_icm(dev, index); 424 } 425 426 int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, 427 int npages, int page_shift, struct mlx4_mr *mr) 428 { 429 u32 index; 430 int err; 431 432 index = mlx4_mr_reserve(dev); 433 if (index == -1) 434 return -ENOMEM; 435 436 err = mlx4_mr_alloc_reserved(dev, index, pd, iova, size, 437 access, npages, page_shift, mr); 438 if (err) 439 mlx4_mr_release(dev, index); 440 441 return err; 442 } 443 EXPORT_SYMBOL_GPL(mlx4_mr_alloc); 444 445 static void mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) 446 { 447 int err; 448 449 if (mr->enabled == MLX4_MR_EN_HW) { 450 err = mlx4_HW2SW_MPT(dev, NULL, 451 key_to_hw_index(mr->key) & 452 (dev->caps.num_mpts - 1)); 453 if (err) 454 mlx4_warn(dev, "xxx HW2SW_MPT failed (%d)\n", err); 455 456 mr->enabled = MLX4_MR_EN_SW; 457 } 458 mlx4_mtt_cleanup(dev, &mr->mtt); 459 } 460 461 void mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) 462 { 463 mlx4_mr_free_reserved(dev, mr); 464 if (mr->enabled) 465 mlx4_mr_free_icm(dev, key_to_hw_index(mr->key)); 466 mlx4_mr_release(dev, key_to_hw_index(mr->key)); 467 } 468 EXPORT_SYMBOL_GPL(mlx4_mr_free); 469 470 int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) 471 { 472 struct mlx4_cmd_mailbox *mailbox; 473 struct mlx4_mpt_entry *mpt_entry; 474 int err; 475 476 err = mlx4_mr_alloc_icm(dev, key_to_hw_index(mr->key)); 477 if (err) 478 return err; 479 480 mailbox = mlx4_alloc_cmd_mailbox(dev); 481 if (IS_ERR(mailbox)) { 482 err = PTR_ERR(mailbox); 483 goto err_table; 484 } 485 mpt_entry = mailbox->buf; 486 487 memset(mpt_entry, 0, sizeof *mpt_entry); 488 489 mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_MIO | 490 MLX4_MPT_FLAG_REGION | 491 mr->access); 492 493 mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); 494 mpt_entry->pd_flags = cpu_to_be32(mr->pd | MLX4_MPT_PD_FLAG_EN_INV); 495 mpt_entry->start = cpu_to_be64(mr->iova); 496 mpt_entry->length = cpu_to_be64(mr->size); 497 mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); 498 499 if (mr->mtt.order < 0) { 500 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); 501 mpt_entry->mtt_addr = 0; 502 } else { 503 mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, 504 &mr->mtt)); 505 } 506 507 if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { 508 /* fast register MR in free state */ 509 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); 510 mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | 511 MLX4_MPT_PD_FLAG_RAE); 512 mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); 513 } else { 514 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); 515 } 516 517 err = mlx4_SW2HW_MPT(dev, mailbox, 518 key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); 519 if (err) { 520 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 521 goto err_cmd; 522 } 523 mr->enabled = MLX4_MR_EN_HW; 524 525 mlx4_free_cmd_mailbox(dev, mailbox); 526 527 return 0; 528 529 err_cmd: 530 mlx4_free_cmd_mailbox(dev, mailbox); 531 532 err_table: 533 mlx4_mr_free_icm(dev, key_to_hw_index(mr->key)); 534 return err; 535 } 536 EXPORT_SYMBOL_GPL(mlx4_mr_enable); 537 538 static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 539 int start_index, int npages, u64 *page_list) 540 { 541 struct mlx4_priv *priv = mlx4_priv(dev); 542 __be64 *mtts; 543 dma_addr_t dma_handle; 544 int i; 545 546 mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->offset + 547 start_index, &dma_handle); 548 549 if (!mtts) 550 return -ENOMEM; 551 552 dma_sync_single_for_cpu(&dev->pdev->dev, dma_handle, 553 npages * sizeof (u64), DMA_TO_DEVICE); 554 555 for (i = 0; i < npages; ++i) 556 mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 557 558 dma_sync_single_for_device(&dev->pdev->dev, dma_handle, 559 npages * sizeof (u64), DMA_TO_DEVICE); 560 561 return 0; 562 } 563 564 int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 565 int start_index, int npages, u64 *page_list) 566 { 567 int err = 0; 568 int chunk; 569 int mtts_per_page; 570 int max_mtts_first_page; 571 572 /* compute how may mtts fit in the first page */ 573 mtts_per_page = PAGE_SIZE / sizeof(u64); 574 max_mtts_first_page = mtts_per_page - (mtt->offset + start_index) 575 % mtts_per_page; 576 577 chunk = min_t(int, max_mtts_first_page, npages); 578 579 while (npages > 0) { 580 err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list); 581 if (err) 582 return err; 583 npages -= chunk; 584 start_index += chunk; 585 page_list += chunk; 586 587 chunk = min_t(int, mtts_per_page, npages); 588 } 589 return err; 590 } 591 592 int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 593 int start_index, int npages, u64 *page_list) 594 { 595 struct mlx4_cmd_mailbox *mailbox = NULL; 596 __be64 *inbox = NULL; 597 int chunk; 598 int err = 0; 599 int i; 600 601 if (mtt->order < 0) 602 return -EINVAL; 603 604 if (mlx4_is_mfunc(dev)) { 605 mailbox = mlx4_alloc_cmd_mailbox(dev); 606 if (IS_ERR(mailbox)) 607 return PTR_ERR(mailbox); 608 inbox = mailbox->buf; 609 610 while (npages > 0) { 611 chunk = min_t(int, MLX4_MAILBOX_SIZE / sizeof(u64) - 2, 612 npages); 613 inbox[0] = cpu_to_be64(mtt->offset + start_index); 614 inbox[1] = 0; 615 for (i = 0; i < chunk; ++i) 616 inbox[i + 2] = cpu_to_be64(page_list[i] | 617 MLX4_MTT_FLAG_PRESENT); 618 err = mlx4_WRITE_MTT(dev, mailbox, chunk); 619 if (err) { 620 mlx4_free_cmd_mailbox(dev, mailbox); 621 return err; 622 } 623 624 npages -= chunk; 625 start_index += chunk; 626 page_list += chunk; 627 } 628 mlx4_free_cmd_mailbox(dev, mailbox); 629 return err; 630 } 631 632 return __mlx4_write_mtt(dev, mtt, start_index, npages, page_list); 633 } 634 EXPORT_SYMBOL_GPL(mlx4_write_mtt); 635 636 int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 637 struct mlx4_buf *buf) 638 { 639 u64 *page_list; 640 int err; 641 int i; 642 643 page_list = kmalloc(buf->npages * sizeof *page_list, GFP_KERNEL); 644 if (!page_list) 645 return -ENOMEM; 646 647 for (i = 0; i < buf->npages; ++i) 648 if (buf->nbufs == 1) 649 page_list[i] = buf->direct.map + (i << buf->page_shift); 650 else 651 page_list[i] = buf->page_list[i].map; 652 653 err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); 654 655 kfree(page_list); 656 return err; 657 } 658 EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); 659 660 int mlx4_init_mr_table(struct mlx4_dev *dev) 661 { 662 struct mlx4_priv *priv = mlx4_priv(dev); 663 struct mlx4_mr_table *mr_table = &priv->mr_table; 664 int err; 665 666 if (!is_power_of_2(dev->caps.num_mpts)) 667 return -EINVAL; 668 669 /* Nothing to do for slaves - all MR handling is forwarded 670 * to the master */ 671 if (mlx4_is_slave(dev)) 672 return 0; 673 674 err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, 675 ~0, dev->caps.reserved_mrws, 0); 676 if (err) 677 return err; 678 679 err = mlx4_buddy_init(&mr_table->mtt_buddy, 680 ilog2((u32)dev->caps.num_mtts / 681 (1 << log_mtts_per_seg))); 682 if (err) 683 goto err_buddy; 684 685 if (dev->caps.reserved_mtts) { 686 priv->reserved_mtts = 687 mlx4_alloc_mtt_range(dev, 688 fls(dev->caps.reserved_mtts - 1)); 689 if (priv->reserved_mtts < 0) { 690 mlx4_warn(dev, "MTT table of order %u is too small.\n", 691 mr_table->mtt_buddy.max_order); 692 err = -ENOMEM; 693 goto err_reserve_mtts; 694 } 695 } 696 697 return 0; 698 699 err_reserve_mtts: 700 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 701 702 err_buddy: 703 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 704 705 return err; 706 } 707 708 void mlx4_cleanup_mr_table(struct mlx4_dev *dev) 709 { 710 struct mlx4_priv *priv = mlx4_priv(dev); 711 struct mlx4_mr_table *mr_table = &priv->mr_table; 712 713 if (mlx4_is_slave(dev)) 714 return; 715 if (priv->reserved_mtts >= 0) 716 mlx4_free_mtt_range(dev, priv->reserved_mtts, 717 fls(dev->caps.reserved_mtts - 1)); 718 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 719 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 720 } 721 722 static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list, 723 int npages, u64 iova) 724 { 725 int i, page_mask; 726 727 if (npages > fmr->max_pages) 728 return -EINVAL; 729 730 page_mask = (1 << fmr->page_shift) - 1; 731 732 /* We are getting page lists, so va must be page aligned. */ 733 if (iova & page_mask) 734 return -EINVAL; 735 736 /* Trust the user not to pass misaligned data in page_list */ 737 if (0) 738 for (i = 0; i < npages; ++i) { 739 if (page_list[i] & ~page_mask) 740 return -EINVAL; 741 } 742 743 if (fmr->maps >= fmr->max_maps) 744 return -EINVAL; 745 746 return 0; 747 } 748 749 int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, 750 int npages, u64 iova, u32 *lkey, u32 *rkey) 751 { 752 u32 key; 753 int i, err; 754 755 err = mlx4_check_fmr(fmr, page_list, npages, iova); 756 if (err) 757 return err; 758 759 ++fmr->maps; 760 761 key = key_to_hw_index(fmr->mr.key); 762 key += dev->caps.num_mpts; 763 *lkey = *rkey = fmr->mr.key = hw_index_to_key(key); 764 765 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; 766 767 /* Make sure MPT status is visible before writing MTT entries */ 768 wmb(); 769 770 dma_sync_single_for_cpu(&dev->pdev->dev, fmr->dma_handle, 771 npages * sizeof(u64), DMA_TO_DEVICE); 772 773 for (i = 0; i < npages; ++i) 774 fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 775 776 dma_sync_single_for_device(&dev->pdev->dev, fmr->dma_handle, 777 npages * sizeof(u64), DMA_TO_DEVICE); 778 779 fmr->mpt->key = cpu_to_be32(key); 780 fmr->mpt->lkey = cpu_to_be32(key); 781 fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift)); 782 fmr->mpt->start = cpu_to_be64(iova); 783 784 /* Make MTT entries are visible before setting MPT status */ 785 wmb(); 786 787 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW; 788 789 /* Make sure MPT status is visible before consumer can use FMR */ 790 wmb(); 791 792 return 0; 793 } 794 EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr); 795 796 int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, 797 int max_maps, u8 page_shift, struct mlx4_fmr *fmr) 798 { 799 struct mlx4_priv *priv = mlx4_priv(dev); 800 int err = -ENOMEM; 801 802 if (max_maps > dev->caps.max_fmr_maps) 803 return -EINVAL; 804 805 if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) 806 return -EINVAL; 807 808 /* All MTTs must fit in the same page */ 809 if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) 810 return -EINVAL; 811 812 fmr->page_shift = page_shift; 813 fmr->max_pages = max_pages; 814 fmr->max_maps = max_maps; 815 fmr->maps = 0; 816 817 err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages, 818 page_shift, &fmr->mr); 819 if (err) 820 return err; 821 822 fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, 823 fmr->mr.mtt.offset, 824 &fmr->dma_handle); 825 826 if (!fmr->mtts) { 827 err = -ENOMEM; 828 goto err_free; 829 } 830 831 return 0; 832 833 err_free: 834 mlx4_mr_free(dev, &fmr->mr); 835 return err; 836 } 837 EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); 838 839 int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 840 { 841 struct mlx4_priv *priv = mlx4_priv(dev); 842 int err; 843 844 err = mlx4_mr_enable(dev, &fmr->mr); 845 if (err) 846 return err; 847 848 fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, 849 key_to_hw_index(fmr->mr.key), NULL); 850 if (!fmr->mpt) 851 return -ENOMEM; 852 853 return 0; 854 } 855 EXPORT_SYMBOL_GPL(mlx4_fmr_enable); 856 857 void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, 858 u32 *lkey, u32 *rkey) 859 { 860 struct mlx4_cmd_mailbox *mailbox; 861 int err; 862 863 if (!fmr->maps) 864 return; 865 866 fmr->maps = 0; 867 868 mailbox = mlx4_alloc_cmd_mailbox(dev); 869 if (IS_ERR(mailbox)) { 870 err = PTR_ERR(mailbox); 871 printk(KERN_WARNING "mlx4_ib: mlx4_alloc_cmd_mailbox" 872 " failed (%d)\n", err); 873 return; 874 } 875 876 err = mlx4_HW2SW_MPT(dev, NULL, 877 key_to_hw_index(fmr->mr.key) & 878 (dev->caps.num_mpts - 1)); 879 mlx4_free_cmd_mailbox(dev, mailbox); 880 if (err) { 881 printk(KERN_WARNING "mlx4_ib: mlx4_HW2SW_MPT failed (%d)\n", 882 err); 883 return; 884 } 885 fmr->mr.enabled = MLX4_MR_EN_SW; 886 } 887 EXPORT_SYMBOL_GPL(mlx4_fmr_unmap); 888 889 int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 890 { 891 if (fmr->maps) 892 return -EBUSY; 893 894 mlx4_mr_free(dev, &fmr->mr); 895 fmr->mr.enabled = MLX4_MR_DISABLED; 896 897 return 0; 898 } 899 EXPORT_SYMBOL_GPL(mlx4_fmr_free); 900 901 int mlx4_SYNC_TPT(struct mlx4_dev *dev) 902 { 903 return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000, 904 MLX4_CMD_NATIVE); 905 } 906 EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT); 907