1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 34 #include <linux/kref.h> 35 #include <linux/random.h> 36 #include <linux/debugfs.h> 37 #include <linux/export.h> 38 #include <linux/delay.h> 39 #include <rdma/ib_umem.h> 40 #include <rdma/ib_umem_odp.h> 41 #include <rdma/ib_verbs.h> 42 #include "mlx5_ib.h" 43 44 enum { 45 MAX_PENDING_REG_MR = 8, 46 }; 47 48 #define MLX5_UMR_ALIGN 2048 49 50 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 51 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 52 static int mr_cache_max_order(struct mlx5_ib_dev *dev); 53 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 54 static bool umr_can_modify_entity_size(struct mlx5_ib_dev *dev) 55 { 56 return !MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled); 57 } 58 59 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) 60 { 61 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled); 62 } 63 64 static bool use_umr(struct mlx5_ib_dev *dev, int order) 65 { 66 return order <= mr_cache_max_order(dev) && 67 umr_can_modify_entity_size(dev); 68 } 69 70 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 71 { 72 int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 73 74 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 75 /* Wait until all page fault handlers using the mr complete. */ 76 if (mr->umem && mr->umem->is_odp) 77 synchronize_srcu(&dev->mr_srcu); 78 #endif 79 80 return err; 81 } 82 83 static int order2idx(struct mlx5_ib_dev *dev, int order) 84 { 85 struct mlx5_mr_cache *cache = &dev->cache; 86 87 if (order < cache->ent[0].order) 88 return 0; 89 else 90 return order - cache->ent[0].order; 91 } 92 93 static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length) 94 { 95 return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >= 96 length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1)); 97 } 98 99 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 100 static void update_odp_mr(struct mlx5_ib_mr *mr) 101 { 102 if (mr->umem->is_odp) { 103 /* 104 * This barrier prevents the compiler from moving the 105 * setting of umem->odp_data->private to point to our 106 * MR, before reg_umr finished, to ensure that the MR 107 * initialization have finished before starting to 108 * handle invalidations. 109 */ 110 smp_wmb(); 111 to_ib_umem_odp(mr->umem)->private = mr; 112 /* 113 * Make sure we will see the new 114 * umem->odp_data->private value in the invalidation 115 * routines, before we can get page faults on the 116 * MR. Page faults can happen once we put the MR in 117 * the tree, below this line. Without the barrier, 118 * there can be a fault handling and an invalidation 119 * before umem->odp_data->private == mr is visible to 120 * the invalidation handler. 121 */ 122 smp_wmb(); 123 } 124 } 125 #endif 126 127 static void reg_mr_callback(int status, void *context) 128 { 129 struct mlx5_ib_mr *mr = context; 130 struct mlx5_ib_dev *dev = mr->dev; 131 struct mlx5_mr_cache *cache = &dev->cache; 132 int c = order2idx(dev, mr->order); 133 struct mlx5_cache_ent *ent = &cache->ent[c]; 134 u8 key; 135 unsigned long flags; 136 struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table; 137 int err; 138 139 spin_lock_irqsave(&ent->lock, flags); 140 ent->pending--; 141 spin_unlock_irqrestore(&ent->lock, flags); 142 if (status) { 143 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status); 144 kfree(mr); 145 dev->fill_delay = 1; 146 mod_timer(&dev->delay_timer, jiffies + HZ); 147 return; 148 } 149 150 mr->mmkey.type = MLX5_MKEY_MR; 151 spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags); 152 key = dev->mdev->priv.mkey_key++; 153 spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags); 154 mr->mmkey.key = mlx5_idx_to_mkey(MLX5_GET(create_mkey_out, mr->out, mkey_index)) | key; 155 156 cache->last_add = jiffies; 157 158 spin_lock_irqsave(&ent->lock, flags); 159 list_add_tail(&mr->list, &ent->head); 160 ent->cur++; 161 ent->size++; 162 spin_unlock_irqrestore(&ent->lock, flags); 163 164 write_lock_irqsave(&table->lock, flags); 165 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmkey.key), 166 &mr->mmkey); 167 if (err) 168 pr_err("Error inserting to mkey tree. 0x%x\n", -err); 169 write_unlock_irqrestore(&table->lock, flags); 170 171 if (!completion_done(&ent->compl)) 172 complete(&ent->compl); 173 } 174 175 static int add_keys(struct mlx5_ib_dev *dev, int c, int num) 176 { 177 struct mlx5_mr_cache *cache = &dev->cache; 178 struct mlx5_cache_ent *ent = &cache->ent[c]; 179 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 180 struct mlx5_ib_mr *mr; 181 void *mkc; 182 u32 *in; 183 int err = 0; 184 int i; 185 186 in = kzalloc(inlen, GFP_KERNEL); 187 if (!in) 188 return -ENOMEM; 189 190 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 191 for (i = 0; i < num; i++) { 192 if (ent->pending >= MAX_PENDING_REG_MR) { 193 err = -EAGAIN; 194 break; 195 } 196 197 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 198 if (!mr) { 199 err = -ENOMEM; 200 break; 201 } 202 mr->order = ent->order; 203 mr->allocated_from_cache = 1; 204 mr->dev = dev; 205 206 MLX5_SET(mkc, mkc, free, 1); 207 MLX5_SET(mkc, mkc, umr_en, 1); 208 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3); 209 MLX5_SET(mkc, mkc, access_mode_4_2, 210 (ent->access_mode >> 2) & 0x7); 211 212 MLX5_SET(mkc, mkc, qpn, 0xffffff); 213 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt); 214 MLX5_SET(mkc, mkc, log_page_size, ent->page); 215 216 spin_lock_irq(&ent->lock); 217 ent->pending++; 218 spin_unlock_irq(&ent->lock); 219 err = mlx5_core_create_mkey_cb(dev->mdev, &mr->mmkey, 220 in, inlen, 221 mr->out, sizeof(mr->out), 222 reg_mr_callback, mr); 223 if (err) { 224 spin_lock_irq(&ent->lock); 225 ent->pending--; 226 spin_unlock_irq(&ent->lock); 227 mlx5_ib_warn(dev, "create mkey failed %d\n", err); 228 kfree(mr); 229 break; 230 } 231 } 232 233 kfree(in); 234 return err; 235 } 236 237 static void remove_keys(struct mlx5_ib_dev *dev, int c, int num) 238 { 239 struct mlx5_mr_cache *cache = &dev->cache; 240 struct mlx5_cache_ent *ent = &cache->ent[c]; 241 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 242 bool odp_mkey_exist = false; 243 #endif 244 struct mlx5_ib_mr *tmp_mr; 245 struct mlx5_ib_mr *mr; 246 LIST_HEAD(del_list); 247 int i; 248 249 for (i = 0; i < num; i++) { 250 spin_lock_irq(&ent->lock); 251 if (list_empty(&ent->head)) { 252 spin_unlock_irq(&ent->lock); 253 break; 254 } 255 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 256 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 257 if (mr->umem && mr->umem->is_odp) 258 odp_mkey_exist = true; 259 #endif 260 list_move(&mr->list, &del_list); 261 ent->cur--; 262 ent->size--; 263 spin_unlock_irq(&ent->lock); 264 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 265 } 266 267 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 268 if (odp_mkey_exist) 269 synchronize_srcu(&dev->mr_srcu); 270 #endif 271 272 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { 273 list_del(&mr->list); 274 kfree(mr); 275 } 276 } 277 278 static ssize_t size_write(struct file *filp, const char __user *buf, 279 size_t count, loff_t *pos) 280 { 281 struct mlx5_cache_ent *ent = filp->private_data; 282 struct mlx5_ib_dev *dev = ent->dev; 283 char lbuf[20] = {0}; 284 u32 var; 285 int err; 286 int c; 287 288 count = min(count, sizeof(lbuf) - 1); 289 if (copy_from_user(lbuf, buf, count)) 290 return -EFAULT; 291 292 c = order2idx(dev, ent->order); 293 294 if (sscanf(lbuf, "%u", &var) != 1) 295 return -EINVAL; 296 297 if (var < ent->limit) 298 return -EINVAL; 299 300 if (var > ent->size) { 301 do { 302 err = add_keys(dev, c, var - ent->size); 303 if (err && err != -EAGAIN) 304 return err; 305 306 usleep_range(3000, 5000); 307 } while (err); 308 } else if (var < ent->size) { 309 remove_keys(dev, c, ent->size - var); 310 } 311 312 return count; 313 } 314 315 static ssize_t size_read(struct file *filp, char __user *buf, size_t count, 316 loff_t *pos) 317 { 318 struct mlx5_cache_ent *ent = filp->private_data; 319 char lbuf[20]; 320 int err; 321 322 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size); 323 if (err < 0) 324 return err; 325 326 return simple_read_from_buffer(buf, count, pos, lbuf, err); 327 } 328 329 static const struct file_operations size_fops = { 330 .owner = THIS_MODULE, 331 .open = simple_open, 332 .write = size_write, 333 .read = size_read, 334 }; 335 336 static ssize_t limit_write(struct file *filp, const char __user *buf, 337 size_t count, loff_t *pos) 338 { 339 struct mlx5_cache_ent *ent = filp->private_data; 340 struct mlx5_ib_dev *dev = ent->dev; 341 char lbuf[20] = {0}; 342 u32 var; 343 int err; 344 int c; 345 346 count = min(count, sizeof(lbuf) - 1); 347 if (copy_from_user(lbuf, buf, count)) 348 return -EFAULT; 349 350 c = order2idx(dev, ent->order); 351 352 if (sscanf(lbuf, "%u", &var) != 1) 353 return -EINVAL; 354 355 if (var > ent->size) 356 return -EINVAL; 357 358 ent->limit = var; 359 360 if (ent->cur < ent->limit) { 361 err = add_keys(dev, c, 2 * ent->limit - ent->cur); 362 if (err) 363 return err; 364 } 365 366 return count; 367 } 368 369 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count, 370 loff_t *pos) 371 { 372 struct mlx5_cache_ent *ent = filp->private_data; 373 char lbuf[20]; 374 int err; 375 376 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); 377 if (err < 0) 378 return err; 379 380 return simple_read_from_buffer(buf, count, pos, lbuf, err); 381 } 382 383 static const struct file_operations limit_fops = { 384 .owner = THIS_MODULE, 385 .open = simple_open, 386 .write = limit_write, 387 .read = limit_read, 388 }; 389 390 static int someone_adding(struct mlx5_mr_cache *cache) 391 { 392 int i; 393 394 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 395 if (cache->ent[i].cur < cache->ent[i].limit) 396 return 1; 397 } 398 399 return 0; 400 } 401 402 static void __cache_work_func(struct mlx5_cache_ent *ent) 403 { 404 struct mlx5_ib_dev *dev = ent->dev; 405 struct mlx5_mr_cache *cache = &dev->cache; 406 int i = order2idx(dev, ent->order); 407 int err; 408 409 if (cache->stopped) 410 return; 411 412 ent = &dev->cache.ent[i]; 413 if (ent->cur < 2 * ent->limit && !dev->fill_delay) { 414 err = add_keys(dev, i, 1); 415 if (ent->cur < 2 * ent->limit) { 416 if (err == -EAGAIN) { 417 mlx5_ib_dbg(dev, "returned eagain, order %d\n", 418 i + 2); 419 queue_delayed_work(cache->wq, &ent->dwork, 420 msecs_to_jiffies(3)); 421 } else if (err) { 422 mlx5_ib_warn(dev, "command failed order %d, err %d\n", 423 i + 2, err); 424 queue_delayed_work(cache->wq, &ent->dwork, 425 msecs_to_jiffies(1000)); 426 } else { 427 queue_work(cache->wq, &ent->work); 428 } 429 } 430 } else if (ent->cur > 2 * ent->limit) { 431 /* 432 * The remove_keys() logic is performed as garbage collection 433 * task. Such task is intended to be run when no other active 434 * processes are running. 435 * 436 * The need_resched() will return TRUE if there are user tasks 437 * to be activated in near future. 438 * 439 * In such case, we don't execute remove_keys() and postpone 440 * the garbage collection work to try to run in next cycle, 441 * in order to free CPU resources to other tasks. 442 */ 443 if (!need_resched() && !someone_adding(cache) && 444 time_after(jiffies, cache->last_add + 300 * HZ)) { 445 remove_keys(dev, i, 1); 446 if (ent->cur > ent->limit) 447 queue_work(cache->wq, &ent->work); 448 } else { 449 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ); 450 } 451 } 452 } 453 454 static void delayed_cache_work_func(struct work_struct *work) 455 { 456 struct mlx5_cache_ent *ent; 457 458 ent = container_of(work, struct mlx5_cache_ent, dwork.work); 459 __cache_work_func(ent); 460 } 461 462 static void cache_work_func(struct work_struct *work) 463 { 464 struct mlx5_cache_ent *ent; 465 466 ent = container_of(work, struct mlx5_cache_ent, work); 467 __cache_work_func(ent); 468 } 469 470 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry) 471 { 472 struct mlx5_mr_cache *cache = &dev->cache; 473 struct mlx5_cache_ent *ent; 474 struct mlx5_ib_mr *mr; 475 int err; 476 477 if (entry < 0 || entry >= MAX_MR_CACHE_ENTRIES) { 478 mlx5_ib_err(dev, "cache entry %d is out of range\n", entry); 479 return NULL; 480 } 481 482 ent = &cache->ent[entry]; 483 while (1) { 484 spin_lock_irq(&ent->lock); 485 if (list_empty(&ent->head)) { 486 spin_unlock_irq(&ent->lock); 487 488 err = add_keys(dev, entry, 1); 489 if (err && err != -EAGAIN) 490 return ERR_PTR(err); 491 492 wait_for_completion(&ent->compl); 493 } else { 494 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, 495 list); 496 list_del(&mr->list); 497 ent->cur--; 498 spin_unlock_irq(&ent->lock); 499 if (ent->cur < ent->limit) 500 queue_work(cache->wq, &ent->work); 501 return mr; 502 } 503 } 504 } 505 506 static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order) 507 { 508 struct mlx5_mr_cache *cache = &dev->cache; 509 struct mlx5_ib_mr *mr = NULL; 510 struct mlx5_cache_ent *ent; 511 int last_umr_cache_entry; 512 int c; 513 int i; 514 515 c = order2idx(dev, order); 516 last_umr_cache_entry = order2idx(dev, mr_cache_max_order(dev)); 517 if (c < 0 || c > last_umr_cache_entry) { 518 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c); 519 return NULL; 520 } 521 522 for (i = c; i <= last_umr_cache_entry; i++) { 523 ent = &cache->ent[i]; 524 525 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i); 526 527 spin_lock_irq(&ent->lock); 528 if (!list_empty(&ent->head)) { 529 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, 530 list); 531 list_del(&mr->list); 532 ent->cur--; 533 spin_unlock_irq(&ent->lock); 534 if (ent->cur < ent->limit) 535 queue_work(cache->wq, &ent->work); 536 break; 537 } 538 spin_unlock_irq(&ent->lock); 539 540 queue_work(cache->wq, &ent->work); 541 } 542 543 if (!mr) 544 cache->ent[c].miss++; 545 546 return mr; 547 } 548 549 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 550 { 551 struct mlx5_mr_cache *cache = &dev->cache; 552 struct mlx5_cache_ent *ent; 553 int shrink = 0; 554 int c; 555 556 if (!mr->allocated_from_cache) 557 return; 558 559 c = order2idx(dev, mr->order); 560 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) { 561 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c); 562 return; 563 } 564 565 if (unreg_umr(dev, mr)) 566 return; 567 568 ent = &cache->ent[c]; 569 spin_lock_irq(&ent->lock); 570 list_add_tail(&mr->list, &ent->head); 571 ent->cur++; 572 if (ent->cur > 2 * ent->limit) 573 shrink = 1; 574 spin_unlock_irq(&ent->lock); 575 576 if (shrink) 577 queue_work(cache->wq, &ent->work); 578 } 579 580 static void clean_keys(struct mlx5_ib_dev *dev, int c) 581 { 582 struct mlx5_mr_cache *cache = &dev->cache; 583 struct mlx5_cache_ent *ent = &cache->ent[c]; 584 bool odp_mkey_exist = false; 585 struct mlx5_ib_mr *tmp_mr; 586 struct mlx5_ib_mr *mr; 587 LIST_HEAD(del_list); 588 589 cancel_delayed_work(&ent->dwork); 590 while (1) { 591 spin_lock_irq(&ent->lock); 592 if (list_empty(&ent->head)) { 593 spin_unlock_irq(&ent->lock); 594 break; 595 } 596 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 597 if (mr->umem && mr->umem->is_odp) 598 odp_mkey_exist = true; 599 list_move(&mr->list, &del_list); 600 ent->cur--; 601 ent->size--; 602 spin_unlock_irq(&ent->lock); 603 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 604 } 605 606 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 607 if (odp_mkey_exist) 608 synchronize_srcu(&dev->mr_srcu); 609 #endif 610 611 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { 612 list_del(&mr->list); 613 kfree(mr); 614 } 615 } 616 617 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) 618 { 619 if (!mlx5_debugfs_root || dev->rep) 620 return; 621 622 debugfs_remove_recursive(dev->cache.root); 623 dev->cache.root = NULL; 624 } 625 626 static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev) 627 { 628 struct mlx5_mr_cache *cache = &dev->cache; 629 struct mlx5_cache_ent *ent; 630 int i; 631 632 if (!mlx5_debugfs_root || dev->rep) 633 return 0; 634 635 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root); 636 if (!cache->root) 637 return -ENOMEM; 638 639 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 640 ent = &cache->ent[i]; 641 sprintf(ent->name, "%d", ent->order); 642 ent->dir = debugfs_create_dir(ent->name, cache->root); 643 if (!ent->dir) 644 goto err; 645 646 ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent, 647 &size_fops); 648 if (!ent->fsize) 649 goto err; 650 651 ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent, 652 &limit_fops); 653 if (!ent->flimit) 654 goto err; 655 656 ent->fcur = debugfs_create_u32("cur", 0400, ent->dir, 657 &ent->cur); 658 if (!ent->fcur) 659 goto err; 660 661 ent->fmiss = debugfs_create_u32("miss", 0600, ent->dir, 662 &ent->miss); 663 if (!ent->fmiss) 664 goto err; 665 } 666 667 return 0; 668 err: 669 mlx5_mr_cache_debugfs_cleanup(dev); 670 671 return -ENOMEM; 672 } 673 674 static void delay_time_func(struct timer_list *t) 675 { 676 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer); 677 678 dev->fill_delay = 0; 679 } 680 681 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) 682 { 683 struct mlx5_mr_cache *cache = &dev->cache; 684 struct mlx5_cache_ent *ent; 685 int err; 686 int i; 687 688 mutex_init(&dev->slow_path_mutex); 689 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM); 690 if (!cache->wq) { 691 mlx5_ib_warn(dev, "failed to create work queue\n"); 692 return -ENOMEM; 693 } 694 695 timer_setup(&dev->delay_timer, delay_time_func, 0); 696 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 697 ent = &cache->ent[i]; 698 INIT_LIST_HEAD(&ent->head); 699 spin_lock_init(&ent->lock); 700 ent->order = i + 2; 701 ent->dev = dev; 702 ent->limit = 0; 703 704 init_completion(&ent->compl); 705 INIT_WORK(&ent->work, cache_work_func); 706 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 707 708 if (i > MR_CACHE_LAST_STD_ENTRY) { 709 mlx5_odp_init_mr_cache_entry(ent); 710 continue; 711 } 712 713 if (ent->order > mr_cache_max_order(dev)) 714 continue; 715 716 ent->page = PAGE_SHIFT; 717 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) / 718 MLX5_IB_UMR_OCTOWORD; 719 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 720 if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) && 721 !dev->rep && 722 mlx5_core_is_pf(dev->mdev)) 723 ent->limit = dev->mdev->profile->mr_cache[i].limit; 724 else 725 ent->limit = 0; 726 queue_work(cache->wq, &ent->work); 727 } 728 729 err = mlx5_mr_cache_debugfs_init(dev); 730 if (err) 731 mlx5_ib_warn(dev, "cache debugfs failure\n"); 732 733 /* 734 * We don't want to fail driver if debugfs failed to initialize, 735 * so we are not forwarding error to the user. 736 */ 737 738 return 0; 739 } 740 741 static void wait_for_async_commands(struct mlx5_ib_dev *dev) 742 { 743 struct mlx5_mr_cache *cache = &dev->cache; 744 struct mlx5_cache_ent *ent; 745 int total = 0; 746 int i; 747 int j; 748 749 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 750 ent = &cache->ent[i]; 751 for (j = 0 ; j < 1000; j++) { 752 if (!ent->pending) 753 break; 754 msleep(50); 755 } 756 } 757 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 758 ent = &cache->ent[i]; 759 total += ent->pending; 760 } 761 762 if (total) 763 mlx5_ib_warn(dev, "aborted while there are %d pending mr requests\n", total); 764 else 765 mlx5_ib_warn(dev, "done with all pending requests\n"); 766 } 767 768 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) 769 { 770 int i; 771 772 if (!dev->cache.wq) 773 return 0; 774 775 dev->cache.stopped = 1; 776 flush_workqueue(dev->cache.wq); 777 778 mlx5_mr_cache_debugfs_cleanup(dev); 779 780 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) 781 clean_keys(dev, i); 782 783 destroy_workqueue(dev->cache.wq); 784 wait_for_async_commands(dev); 785 del_timer_sync(&dev->delay_timer); 786 787 return 0; 788 } 789 790 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) 791 { 792 struct mlx5_ib_dev *dev = to_mdev(pd->device); 793 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 794 struct mlx5_core_dev *mdev = dev->mdev; 795 struct mlx5_ib_mr *mr; 796 void *mkc; 797 u32 *in; 798 int err; 799 800 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 801 if (!mr) 802 return ERR_PTR(-ENOMEM); 803 804 in = kzalloc(inlen, GFP_KERNEL); 805 if (!in) { 806 err = -ENOMEM; 807 goto err_free; 808 } 809 810 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 811 812 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 813 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 814 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 815 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 816 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 817 MLX5_SET(mkc, mkc, lr, 1); 818 819 MLX5_SET(mkc, mkc, length64, 1); 820 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 821 MLX5_SET(mkc, mkc, qpn, 0xffffff); 822 MLX5_SET64(mkc, mkc, start_addr, 0); 823 824 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen); 825 if (err) 826 goto err_in; 827 828 kfree(in); 829 mr->mmkey.type = MLX5_MKEY_MR; 830 mr->ibmr.lkey = mr->mmkey.key; 831 mr->ibmr.rkey = mr->mmkey.key; 832 mr->umem = NULL; 833 834 return &mr->ibmr; 835 836 err_in: 837 kfree(in); 838 839 err_free: 840 kfree(mr); 841 842 return ERR_PTR(err); 843 } 844 845 static int get_octo_len(u64 addr, u64 len, int page_shift) 846 { 847 u64 page_size = 1ULL << page_shift; 848 u64 offset; 849 int npages; 850 851 offset = addr & (page_size - 1); 852 npages = ALIGN(len + offset, page_size) >> page_shift; 853 return (npages + 1) / 2; 854 } 855 856 static int mr_cache_max_order(struct mlx5_ib_dev *dev) 857 { 858 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 859 return MR_CACHE_LAST_STD_ENTRY + 2; 860 return MLX5_MAX_UMR_SHIFT; 861 } 862 863 static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length, 864 int access_flags, struct ib_umem **umem, 865 int *npages, int *page_shift, int *ncont, 866 int *order) 867 { 868 struct mlx5_ib_dev *dev = to_mdev(pd->device); 869 struct ib_umem *u; 870 int err; 871 872 *umem = NULL; 873 874 u = ib_umem_get(pd->uobject->context, start, length, access_flags, 0); 875 err = PTR_ERR_OR_ZERO(u); 876 if (err) { 877 mlx5_ib_dbg(dev, "umem get failed (%d)\n", err); 878 return err; 879 } 880 881 mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages, 882 page_shift, ncont, order); 883 if (!*npages) { 884 mlx5_ib_warn(dev, "avoid zero region\n"); 885 ib_umem_release(u); 886 return -EINVAL; 887 } 888 889 *umem = u; 890 891 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n", 892 *npages, *ncont, *order, *page_shift); 893 894 return 0; 895 } 896 897 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc) 898 { 899 struct mlx5_ib_umr_context *context = 900 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe); 901 902 context->status = wc->status; 903 complete(&context->done); 904 } 905 906 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context) 907 { 908 context->cqe.done = mlx5_ib_umr_done; 909 context->status = -1; 910 init_completion(&context->done); 911 } 912 913 static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev, 914 struct mlx5_umr_wr *umrwr) 915 { 916 struct umr_common *umrc = &dev->umrc; 917 const struct ib_send_wr *bad; 918 int err; 919 struct mlx5_ib_umr_context umr_context; 920 921 mlx5_ib_init_umr_context(&umr_context); 922 umrwr->wr.wr_cqe = &umr_context.cqe; 923 924 down(&umrc->sem); 925 err = ib_post_send(umrc->qp, &umrwr->wr, &bad); 926 if (err) { 927 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err); 928 } else { 929 wait_for_completion(&umr_context.done); 930 if (umr_context.status != IB_WC_SUCCESS) { 931 mlx5_ib_warn(dev, "reg umr failed (%u)\n", 932 umr_context.status); 933 err = -EFAULT; 934 } 935 } 936 up(&umrc->sem); 937 return err; 938 } 939 940 static struct mlx5_ib_mr *alloc_mr_from_cache( 941 struct ib_pd *pd, struct ib_umem *umem, 942 u64 virt_addr, u64 len, int npages, 943 int page_shift, int order, int access_flags) 944 { 945 struct mlx5_ib_dev *dev = to_mdev(pd->device); 946 struct mlx5_ib_mr *mr; 947 int err = 0; 948 int i; 949 950 for (i = 0; i < 1; i++) { 951 mr = alloc_cached_mr(dev, order); 952 if (mr) 953 break; 954 955 err = add_keys(dev, order2idx(dev, order), 1); 956 if (err && err != -EAGAIN) { 957 mlx5_ib_warn(dev, "add_keys failed, err %d\n", err); 958 break; 959 } 960 } 961 962 if (!mr) 963 return ERR_PTR(-EAGAIN); 964 965 mr->ibmr.pd = pd; 966 mr->umem = umem; 967 mr->access_flags = access_flags; 968 mr->desc_size = sizeof(struct mlx5_mtt); 969 mr->mmkey.iova = virt_addr; 970 mr->mmkey.size = len; 971 mr->mmkey.pd = to_mpd(pd)->pdn; 972 973 return mr; 974 } 975 976 static inline int populate_xlt(struct mlx5_ib_mr *mr, int idx, int npages, 977 void *xlt, int page_shift, size_t size, 978 int flags) 979 { 980 struct mlx5_ib_dev *dev = mr->dev; 981 struct ib_umem *umem = mr->umem; 982 983 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 984 if (!umr_can_use_indirect_mkey(dev)) 985 return -EPERM; 986 mlx5_odp_populate_klm(xlt, idx, npages, mr, flags); 987 return npages; 988 } 989 990 npages = min_t(size_t, npages, ib_umem_num_pages(umem) - idx); 991 992 if (!(flags & MLX5_IB_UPD_XLT_ZAP)) { 993 __mlx5_ib_populate_pas(dev, umem, page_shift, 994 idx, npages, xlt, 995 MLX5_IB_MTT_PRESENT); 996 /* Clear padding after the pages 997 * brought from the umem. 998 */ 999 memset(xlt + (npages * sizeof(struct mlx5_mtt)), 0, 1000 size - npages * sizeof(struct mlx5_mtt)); 1001 } 1002 1003 return npages; 1004 } 1005 1006 #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \ 1007 MLX5_UMR_MTT_ALIGNMENT) 1008 #define MLX5_SPARE_UMR_CHUNK 0x10000 1009 1010 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, 1011 int page_shift, int flags) 1012 { 1013 struct mlx5_ib_dev *dev = mr->dev; 1014 struct device *ddev = dev->ib_dev.dev.parent; 1015 int size; 1016 void *xlt; 1017 dma_addr_t dma; 1018 struct mlx5_umr_wr wr; 1019 struct ib_sge sg; 1020 int err = 0; 1021 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) 1022 ? sizeof(struct mlx5_klm) 1023 : sizeof(struct mlx5_mtt); 1024 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size; 1025 const int page_mask = page_align - 1; 1026 size_t pages_mapped = 0; 1027 size_t pages_to_map = 0; 1028 size_t pages_iter = 0; 1029 gfp_t gfp; 1030 bool use_emergency_page = false; 1031 1032 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) && 1033 !umr_can_use_indirect_mkey(dev)) 1034 return -EPERM; 1035 1036 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes, 1037 * so we need to align the offset and length accordingly 1038 */ 1039 if (idx & page_mask) { 1040 npages += idx & page_mask; 1041 idx &= ~page_mask; 1042 } 1043 1044 gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL; 1045 gfp |= __GFP_ZERO | __GFP_NOWARN; 1046 1047 pages_to_map = ALIGN(npages, page_align); 1048 size = desc_size * pages_to_map; 1049 size = min_t(int, size, MLX5_MAX_UMR_CHUNK); 1050 1051 xlt = (void *)__get_free_pages(gfp, get_order(size)); 1052 if (!xlt && size > MLX5_SPARE_UMR_CHUNK) { 1053 mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n", 1054 size, get_order(size), MLX5_SPARE_UMR_CHUNK); 1055 1056 size = MLX5_SPARE_UMR_CHUNK; 1057 xlt = (void *)__get_free_pages(gfp, get_order(size)); 1058 } 1059 1060 if (!xlt) { 1061 mlx5_ib_warn(dev, "Using XLT emergency buffer\n"); 1062 xlt = (void *)mlx5_ib_get_xlt_emergency_page(); 1063 size = PAGE_SIZE; 1064 memset(xlt, 0, size); 1065 use_emergency_page = true; 1066 } 1067 pages_iter = size / desc_size; 1068 dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE); 1069 if (dma_mapping_error(ddev, dma)) { 1070 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n"); 1071 err = -ENOMEM; 1072 goto free_xlt; 1073 } 1074 1075 sg.addr = dma; 1076 sg.lkey = dev->umrc.pd->local_dma_lkey; 1077 1078 memset(&wr, 0, sizeof(wr)); 1079 wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT; 1080 if (!(flags & MLX5_IB_UPD_XLT_ENABLE)) 1081 wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1082 wr.wr.sg_list = &sg; 1083 wr.wr.num_sge = 1; 1084 wr.wr.opcode = MLX5_IB_WR_UMR; 1085 1086 wr.pd = mr->ibmr.pd; 1087 wr.mkey = mr->mmkey.key; 1088 wr.length = mr->mmkey.size; 1089 wr.virt_addr = mr->mmkey.iova; 1090 wr.access_flags = mr->access_flags; 1091 wr.page_shift = page_shift; 1092 1093 for (pages_mapped = 0; 1094 pages_mapped < pages_to_map && !err; 1095 pages_mapped += pages_iter, idx += pages_iter) { 1096 npages = min_t(int, pages_iter, pages_to_map - pages_mapped); 1097 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE); 1098 npages = populate_xlt(mr, idx, npages, xlt, 1099 page_shift, size, flags); 1100 1101 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE); 1102 1103 sg.length = ALIGN(npages * desc_size, 1104 MLX5_UMR_MTT_ALIGNMENT); 1105 1106 if (pages_mapped + pages_iter >= pages_to_map) { 1107 if (flags & MLX5_IB_UPD_XLT_ENABLE) 1108 wr.wr.send_flags |= 1109 MLX5_IB_SEND_UMR_ENABLE_MR | 1110 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS | 1111 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1112 if (flags & MLX5_IB_UPD_XLT_PD || 1113 flags & MLX5_IB_UPD_XLT_ACCESS) 1114 wr.wr.send_flags |= 1115 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1116 if (flags & MLX5_IB_UPD_XLT_ADDR) 1117 wr.wr.send_flags |= 1118 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1119 } 1120 1121 wr.offset = idx * desc_size; 1122 wr.xlt_size = sg.length; 1123 1124 err = mlx5_ib_post_send_wait(dev, &wr); 1125 } 1126 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE); 1127 1128 free_xlt: 1129 if (use_emergency_page) 1130 mlx5_ib_put_xlt_emergency_page(); 1131 else 1132 free_pages((unsigned long)xlt, get_order(size)); 1133 1134 return err; 1135 } 1136 1137 /* 1138 * If ibmr is NULL it will be allocated by reg_create. 1139 * Else, the given ibmr will be used. 1140 */ 1141 static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, 1142 u64 virt_addr, u64 length, 1143 struct ib_umem *umem, int npages, 1144 int page_shift, int access_flags, 1145 bool populate) 1146 { 1147 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1148 struct mlx5_ib_mr *mr; 1149 __be64 *pas; 1150 void *mkc; 1151 int inlen; 1152 u32 *in; 1153 int err; 1154 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 1155 1156 mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL); 1157 if (!mr) 1158 return ERR_PTR(-ENOMEM); 1159 1160 mr->ibmr.pd = pd; 1161 mr->access_flags = access_flags; 1162 1163 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1164 if (populate) 1165 inlen += sizeof(*pas) * roundup(npages, 2); 1166 in = kvzalloc(inlen, GFP_KERNEL); 1167 if (!in) { 1168 err = -ENOMEM; 1169 goto err_1; 1170 } 1171 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 1172 if (populate && !(access_flags & IB_ACCESS_ON_DEMAND)) 1173 mlx5_ib_populate_pas(dev, umem, page_shift, pas, 1174 pg_cap ? MLX5_IB_MTT_PRESENT : 0); 1175 1176 /* The pg_access bit allows setting the access flags 1177 * in the page list submitted with the command. */ 1178 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap)); 1179 1180 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1181 MLX5_SET(mkc, mkc, free, !populate); 1182 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 1183 MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); 1184 MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE)); 1185 MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ)); 1186 MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE)); 1187 MLX5_SET(mkc, mkc, lr, 1); 1188 MLX5_SET(mkc, mkc, umr_en, 1); 1189 1190 MLX5_SET64(mkc, mkc, start_addr, virt_addr); 1191 MLX5_SET64(mkc, mkc, len, length); 1192 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1193 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 1194 MLX5_SET(mkc, mkc, translations_octword_size, 1195 get_octo_len(virt_addr, length, page_shift)); 1196 MLX5_SET(mkc, mkc, log_page_size, page_shift); 1197 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1198 if (populate) { 1199 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 1200 get_octo_len(virt_addr, length, page_shift)); 1201 } 1202 1203 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); 1204 if (err) { 1205 mlx5_ib_warn(dev, "create mkey failed\n"); 1206 goto err_2; 1207 } 1208 mr->mmkey.type = MLX5_MKEY_MR; 1209 mr->desc_size = sizeof(struct mlx5_mtt); 1210 mr->dev = dev; 1211 kvfree(in); 1212 1213 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key); 1214 1215 return mr; 1216 1217 err_2: 1218 kvfree(in); 1219 1220 err_1: 1221 if (!ibmr) 1222 kfree(mr); 1223 1224 return ERR_PTR(err); 1225 } 1226 1227 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 1228 int npages, u64 length, int access_flags) 1229 { 1230 mr->npages = npages; 1231 atomic_add(npages, &dev->mdev->priv.reg_pages); 1232 mr->ibmr.lkey = mr->mmkey.key; 1233 mr->ibmr.rkey = mr->mmkey.key; 1234 mr->ibmr.length = length; 1235 mr->access_flags = access_flags; 1236 } 1237 1238 static struct ib_mr *mlx5_ib_get_memic_mr(struct ib_pd *pd, u64 memic_addr, 1239 u64 length, int acc) 1240 { 1241 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1242 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1243 struct mlx5_core_dev *mdev = dev->mdev; 1244 struct mlx5_ib_mr *mr; 1245 void *mkc; 1246 u32 *in; 1247 int err; 1248 1249 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1250 if (!mr) 1251 return ERR_PTR(-ENOMEM); 1252 1253 in = kzalloc(inlen, GFP_KERNEL); 1254 if (!in) { 1255 err = -ENOMEM; 1256 goto err_free; 1257 } 1258 1259 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1260 1261 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MEMIC & 0x3); 1262 MLX5_SET(mkc, mkc, access_mode_4_2, 1263 (MLX5_MKC_ACCESS_MODE_MEMIC >> 2) & 0x7); 1264 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 1265 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 1266 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 1267 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 1268 MLX5_SET(mkc, mkc, lr, 1); 1269 1270 MLX5_SET64(mkc, mkc, len, length); 1271 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1272 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1273 MLX5_SET64(mkc, mkc, start_addr, 1274 memic_addr - pci_resource_start(dev->mdev->pdev, 0)); 1275 1276 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen); 1277 if (err) 1278 goto err_in; 1279 1280 kfree(in); 1281 1282 mr->umem = NULL; 1283 set_mr_fields(dev, mr, 0, length, acc); 1284 1285 return &mr->ibmr; 1286 1287 err_in: 1288 kfree(in); 1289 1290 err_free: 1291 kfree(mr); 1292 1293 return ERR_PTR(err); 1294 } 1295 1296 int mlx5_ib_advise_mr(struct ib_pd *pd, 1297 enum ib_uverbs_advise_mr_advice advice, 1298 u32 flags, 1299 struct ib_sge *sg_list, 1300 u32 num_sge, 1301 struct uverbs_attr_bundle *attrs) 1302 { 1303 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && 1304 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE) 1305 return -EOPNOTSUPP; 1306 1307 return mlx5_ib_advise_mr_prefetch(pd, advice, flags, 1308 sg_list, num_sge); 1309 } 1310 1311 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, 1312 struct ib_dm_mr_attr *attr, 1313 struct uverbs_attr_bundle *attrs) 1314 { 1315 struct mlx5_ib_dm *mdm = to_mdm(dm); 1316 u64 memic_addr; 1317 1318 if (attr->access_flags & ~MLX5_IB_DM_ALLOWED_ACCESS) 1319 return ERR_PTR(-EINVAL); 1320 1321 memic_addr = mdm->dev_addr + attr->offset; 1322 1323 return mlx5_ib_get_memic_mr(pd, memic_addr, attr->length, 1324 attr->access_flags); 1325 } 1326 1327 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 1328 u64 virt_addr, int access_flags, 1329 struct ib_udata *udata) 1330 { 1331 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1332 struct mlx5_ib_mr *mr = NULL; 1333 bool populate_mtts = false; 1334 struct ib_umem *umem; 1335 int page_shift; 1336 int npages; 1337 int ncont; 1338 int order; 1339 int err; 1340 1341 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) 1342 return ERR_PTR(-EOPNOTSUPP); 1343 1344 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1345 start, virt_addr, length, access_flags); 1346 1347 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1348 if (!start && length == U64_MAX) { 1349 if (!(access_flags & IB_ACCESS_ON_DEMAND) || 1350 !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1351 return ERR_PTR(-EINVAL); 1352 1353 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), access_flags); 1354 if (IS_ERR(mr)) 1355 return ERR_CAST(mr); 1356 return &mr->ibmr; 1357 } 1358 #endif 1359 1360 err = mr_umem_get(pd, start, length, access_flags, &umem, &npages, 1361 &page_shift, &ncont, &order); 1362 1363 if (err < 0) 1364 return ERR_PTR(err); 1365 1366 if (use_umr(dev, order)) { 1367 mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont, 1368 page_shift, order, access_flags); 1369 if (PTR_ERR(mr) == -EAGAIN) { 1370 mlx5_ib_dbg(dev, "cache empty for order %d\n", order); 1371 mr = NULL; 1372 } 1373 populate_mtts = false; 1374 } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) { 1375 if (access_flags & IB_ACCESS_ON_DEMAND) { 1376 err = -EINVAL; 1377 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n"); 1378 goto error; 1379 } 1380 populate_mtts = true; 1381 } 1382 1383 if (!mr) { 1384 if (!umr_can_modify_entity_size(dev)) 1385 populate_mtts = true; 1386 mutex_lock(&dev->slow_path_mutex); 1387 mr = reg_create(NULL, pd, virt_addr, length, umem, ncont, 1388 page_shift, access_flags, populate_mtts); 1389 mutex_unlock(&dev->slow_path_mutex); 1390 } 1391 1392 if (IS_ERR(mr)) { 1393 err = PTR_ERR(mr); 1394 goto error; 1395 } 1396 1397 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); 1398 1399 mr->umem = umem; 1400 set_mr_fields(dev, mr, npages, length, access_flags); 1401 1402 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1403 update_odp_mr(mr); 1404 #endif 1405 1406 if (!populate_mtts) { 1407 int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE; 1408 1409 if (access_flags & IB_ACCESS_ON_DEMAND) 1410 update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP; 1411 1412 err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift, 1413 update_xlt_flags); 1414 1415 if (err) { 1416 dereg_mr(dev, mr); 1417 return ERR_PTR(err); 1418 } 1419 } 1420 1421 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1422 mr->live = 1; 1423 #endif 1424 return &mr->ibmr; 1425 error: 1426 ib_umem_release(umem); 1427 return ERR_PTR(err); 1428 } 1429 1430 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1431 { 1432 struct mlx5_core_dev *mdev = dev->mdev; 1433 struct mlx5_umr_wr umrwr = {}; 1434 1435 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 1436 return 0; 1437 1438 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR | 1439 MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1440 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1441 umrwr.mkey = mr->mmkey.key; 1442 1443 return mlx5_ib_post_send_wait(dev, &umrwr); 1444 } 1445 1446 static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1447 int access_flags, int flags) 1448 { 1449 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1450 struct mlx5_umr_wr umrwr = {}; 1451 int err; 1452 1453 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1454 1455 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1456 umrwr.mkey = mr->mmkey.key; 1457 1458 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) { 1459 umrwr.pd = pd; 1460 umrwr.access_flags = access_flags; 1461 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1462 } 1463 1464 err = mlx5_ib_post_send_wait(dev, &umrwr); 1465 1466 return err; 1467 } 1468 1469 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 1470 u64 length, u64 virt_addr, int new_access_flags, 1471 struct ib_pd *new_pd, struct ib_udata *udata) 1472 { 1473 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device); 1474 struct mlx5_ib_mr *mr = to_mmr(ib_mr); 1475 struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd; 1476 int access_flags = flags & IB_MR_REREG_ACCESS ? 1477 new_access_flags : 1478 mr->access_flags; 1479 int page_shift = 0; 1480 int upd_flags = 0; 1481 int npages = 0; 1482 int ncont = 0; 1483 int order = 0; 1484 u64 addr, len; 1485 int err; 1486 1487 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1488 start, virt_addr, length, access_flags); 1489 1490 atomic_sub(mr->npages, &dev->mdev->priv.reg_pages); 1491 1492 if (!mr->umem) 1493 return -EINVAL; 1494 1495 if (flags & IB_MR_REREG_TRANS) { 1496 addr = virt_addr; 1497 len = length; 1498 } else { 1499 addr = mr->umem->address; 1500 len = mr->umem->length; 1501 } 1502 1503 if (flags != IB_MR_REREG_PD) { 1504 /* 1505 * Replace umem. This needs to be done whether or not UMR is 1506 * used. 1507 */ 1508 flags |= IB_MR_REREG_TRANS; 1509 ib_umem_release(mr->umem); 1510 mr->umem = NULL; 1511 err = mr_umem_get(pd, addr, len, access_flags, &mr->umem, 1512 &npages, &page_shift, &ncont, &order); 1513 if (err) 1514 goto err; 1515 } 1516 1517 if (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len)) { 1518 /* 1519 * UMR can't be used - MKey needs to be replaced. 1520 */ 1521 if (mr->allocated_from_cache) 1522 err = unreg_umr(dev, mr); 1523 else 1524 err = destroy_mkey(dev, mr); 1525 if (err) 1526 goto err; 1527 1528 mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont, 1529 page_shift, access_flags, true); 1530 1531 if (IS_ERR(mr)) { 1532 err = PTR_ERR(mr); 1533 mr = to_mmr(ib_mr); 1534 goto err; 1535 } 1536 1537 mr->allocated_from_cache = 0; 1538 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1539 mr->live = 1; 1540 #endif 1541 } else { 1542 /* 1543 * Send a UMR WQE 1544 */ 1545 mr->ibmr.pd = pd; 1546 mr->access_flags = access_flags; 1547 mr->mmkey.iova = addr; 1548 mr->mmkey.size = len; 1549 mr->mmkey.pd = to_mpd(pd)->pdn; 1550 1551 if (flags & IB_MR_REREG_TRANS) { 1552 upd_flags = MLX5_IB_UPD_XLT_ADDR; 1553 if (flags & IB_MR_REREG_PD) 1554 upd_flags |= MLX5_IB_UPD_XLT_PD; 1555 if (flags & IB_MR_REREG_ACCESS) 1556 upd_flags |= MLX5_IB_UPD_XLT_ACCESS; 1557 err = mlx5_ib_update_xlt(mr, 0, npages, page_shift, 1558 upd_flags); 1559 } else { 1560 err = rereg_umr(pd, mr, access_flags, flags); 1561 } 1562 1563 if (err) 1564 goto err; 1565 } 1566 1567 set_mr_fields(dev, mr, npages, len, access_flags); 1568 1569 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1570 update_odp_mr(mr); 1571 #endif 1572 return 0; 1573 1574 err: 1575 if (mr->umem) { 1576 ib_umem_release(mr->umem); 1577 mr->umem = NULL; 1578 } 1579 clean_mr(dev, mr); 1580 return err; 1581 } 1582 1583 static int 1584 mlx5_alloc_priv_descs(struct ib_device *device, 1585 struct mlx5_ib_mr *mr, 1586 int ndescs, 1587 int desc_size) 1588 { 1589 int size = ndescs * desc_size; 1590 int add_size; 1591 int ret; 1592 1593 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0); 1594 1595 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL); 1596 if (!mr->descs_alloc) 1597 return -ENOMEM; 1598 1599 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN); 1600 1601 mr->desc_map = dma_map_single(device->dev.parent, mr->descs, 1602 size, DMA_TO_DEVICE); 1603 if (dma_mapping_error(device->dev.parent, mr->desc_map)) { 1604 ret = -ENOMEM; 1605 goto err; 1606 } 1607 1608 return 0; 1609 err: 1610 kfree(mr->descs_alloc); 1611 1612 return ret; 1613 } 1614 1615 static void 1616 mlx5_free_priv_descs(struct mlx5_ib_mr *mr) 1617 { 1618 if (mr->descs) { 1619 struct ib_device *device = mr->ibmr.device; 1620 int size = mr->max_descs * mr->desc_size; 1621 1622 dma_unmap_single(device->dev.parent, mr->desc_map, 1623 size, DMA_TO_DEVICE); 1624 kfree(mr->descs_alloc); 1625 mr->descs = NULL; 1626 } 1627 } 1628 1629 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1630 { 1631 int allocated_from_cache = mr->allocated_from_cache; 1632 1633 if (mr->sig) { 1634 if (mlx5_core_destroy_psv(dev->mdev, 1635 mr->sig->psv_memory.psv_idx)) 1636 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1637 mr->sig->psv_memory.psv_idx); 1638 if (mlx5_core_destroy_psv(dev->mdev, 1639 mr->sig->psv_wire.psv_idx)) 1640 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1641 mr->sig->psv_wire.psv_idx); 1642 kfree(mr->sig); 1643 mr->sig = NULL; 1644 } 1645 1646 mlx5_free_priv_descs(mr); 1647 1648 if (!allocated_from_cache) 1649 destroy_mkey(dev, mr); 1650 } 1651 1652 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1653 { 1654 int npages = mr->npages; 1655 struct ib_umem *umem = mr->umem; 1656 1657 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1658 if (umem && umem->is_odp) { 1659 struct ib_umem_odp *umem_odp = to_ib_umem_odp(umem); 1660 1661 /* Prevent new page faults from succeeding */ 1662 mr->live = 0; 1663 /* Wait for all running page-fault handlers to finish. */ 1664 synchronize_srcu(&dev->mr_srcu); 1665 /* Destroy all page mappings */ 1666 if (umem_odp->page_list) 1667 mlx5_ib_invalidate_range(umem_odp, ib_umem_start(umem), 1668 ib_umem_end(umem)); 1669 else 1670 mlx5_ib_free_implicit_mr(mr); 1671 /* 1672 * We kill the umem before the MR for ODP, 1673 * so that there will not be any invalidations in 1674 * flight, looking at the *mr struct. 1675 */ 1676 ib_umem_release(umem); 1677 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1678 1679 /* Avoid double-freeing the umem. */ 1680 umem = NULL; 1681 } 1682 #endif 1683 clean_mr(dev, mr); 1684 1685 /* 1686 * We should unregister the DMA address from the HCA before 1687 * remove the DMA mapping. 1688 */ 1689 mlx5_mr_cache_free(dev, mr); 1690 if (umem) { 1691 ib_umem_release(umem); 1692 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1693 } 1694 if (!mr->allocated_from_cache) 1695 kfree(mr); 1696 } 1697 1698 int mlx5_ib_dereg_mr(struct ib_mr *ibmr) 1699 { 1700 dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr)); 1701 return 0; 1702 } 1703 1704 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, 1705 enum ib_mr_type mr_type, 1706 u32 max_num_sg) 1707 { 1708 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1709 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1710 int ndescs = ALIGN(max_num_sg, 4); 1711 struct mlx5_ib_mr *mr; 1712 void *mkc; 1713 u32 *in; 1714 int err; 1715 1716 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1717 if (!mr) 1718 return ERR_PTR(-ENOMEM); 1719 1720 in = kzalloc(inlen, GFP_KERNEL); 1721 if (!in) { 1722 err = -ENOMEM; 1723 goto err_free; 1724 } 1725 1726 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1727 MLX5_SET(mkc, mkc, free, 1); 1728 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 1729 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1730 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1731 1732 if (mr_type == IB_MR_TYPE_MEM_REG) { 1733 mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1734 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 1735 err = mlx5_alloc_priv_descs(pd->device, mr, 1736 ndescs, sizeof(struct mlx5_mtt)); 1737 if (err) 1738 goto err_free_in; 1739 1740 mr->desc_size = sizeof(struct mlx5_mtt); 1741 mr->max_descs = ndescs; 1742 } else if (mr_type == IB_MR_TYPE_SG_GAPS) { 1743 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; 1744 1745 err = mlx5_alloc_priv_descs(pd->device, mr, 1746 ndescs, sizeof(struct mlx5_klm)); 1747 if (err) 1748 goto err_free_in; 1749 mr->desc_size = sizeof(struct mlx5_klm); 1750 mr->max_descs = ndescs; 1751 } else if (mr_type == IB_MR_TYPE_SIGNATURE) { 1752 u32 psv_index[2]; 1753 1754 MLX5_SET(mkc, mkc, bsf_en, 1); 1755 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); 1756 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); 1757 if (!mr->sig) { 1758 err = -ENOMEM; 1759 goto err_free_in; 1760 } 1761 1762 /* create mem & wire PSVs */ 1763 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 1764 2, psv_index); 1765 if (err) 1766 goto err_free_sig; 1767 1768 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; 1769 mr->sig->psv_memory.psv_idx = psv_index[0]; 1770 mr->sig->psv_wire.psv_idx = psv_index[1]; 1771 1772 mr->sig->sig_status_checked = true; 1773 mr->sig->sig_err_exists = false; 1774 /* Next UMR, Arm SIGERR */ 1775 ++mr->sig->sigerr_count; 1776 } else { 1777 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); 1778 err = -EINVAL; 1779 goto err_free_in; 1780 } 1781 1782 MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3); 1783 MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7); 1784 MLX5_SET(mkc, mkc, umr_en, 1); 1785 1786 mr->ibmr.device = pd->device; 1787 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); 1788 if (err) 1789 goto err_destroy_psv; 1790 1791 mr->mmkey.type = MLX5_MKEY_MR; 1792 mr->ibmr.lkey = mr->mmkey.key; 1793 mr->ibmr.rkey = mr->mmkey.key; 1794 mr->umem = NULL; 1795 kfree(in); 1796 1797 return &mr->ibmr; 1798 1799 err_destroy_psv: 1800 if (mr->sig) { 1801 if (mlx5_core_destroy_psv(dev->mdev, 1802 mr->sig->psv_memory.psv_idx)) 1803 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1804 mr->sig->psv_memory.psv_idx); 1805 if (mlx5_core_destroy_psv(dev->mdev, 1806 mr->sig->psv_wire.psv_idx)) 1807 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1808 mr->sig->psv_wire.psv_idx); 1809 } 1810 mlx5_free_priv_descs(mr); 1811 err_free_sig: 1812 kfree(mr->sig); 1813 err_free_in: 1814 kfree(in); 1815 err_free: 1816 kfree(mr); 1817 return ERR_PTR(err); 1818 } 1819 1820 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, 1821 struct ib_udata *udata) 1822 { 1823 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1824 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1825 struct mlx5_ib_mw *mw = NULL; 1826 u32 *in = NULL; 1827 void *mkc; 1828 int ndescs; 1829 int err; 1830 struct mlx5_ib_alloc_mw req = {}; 1831 struct { 1832 __u32 comp_mask; 1833 __u32 response_length; 1834 } resp = {}; 1835 1836 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req))); 1837 if (err) 1838 return ERR_PTR(err); 1839 1840 if (req.comp_mask || req.reserved1 || req.reserved2) 1841 return ERR_PTR(-EOPNOTSUPP); 1842 1843 if (udata->inlen > sizeof(req) && 1844 !ib_is_udata_cleared(udata, sizeof(req), 1845 udata->inlen - sizeof(req))) 1846 return ERR_PTR(-EOPNOTSUPP); 1847 1848 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4); 1849 1850 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 1851 in = kzalloc(inlen, GFP_KERNEL); 1852 if (!mw || !in) { 1853 err = -ENOMEM; 1854 goto free; 1855 } 1856 1857 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1858 1859 MLX5_SET(mkc, mkc, free, 1); 1860 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 1861 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1862 MLX5_SET(mkc, mkc, umr_en, 1); 1863 MLX5_SET(mkc, mkc, lr, 1); 1864 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 1865 MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2))); 1866 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1867 1868 err = mlx5_core_create_mkey(dev->mdev, &mw->mmkey, in, inlen); 1869 if (err) 1870 goto free; 1871 1872 mw->mmkey.type = MLX5_MKEY_MW; 1873 mw->ibmw.rkey = mw->mmkey.key; 1874 mw->ndescs = ndescs; 1875 1876 resp.response_length = min(offsetof(typeof(resp), response_length) + 1877 sizeof(resp.response_length), udata->outlen); 1878 if (resp.response_length) { 1879 err = ib_copy_to_udata(udata, &resp, resp.response_length); 1880 if (err) { 1881 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey); 1882 goto free; 1883 } 1884 } 1885 1886 kfree(in); 1887 return &mw->ibmw; 1888 1889 free: 1890 kfree(mw); 1891 kfree(in); 1892 return ERR_PTR(err); 1893 } 1894 1895 int mlx5_ib_dealloc_mw(struct ib_mw *mw) 1896 { 1897 struct mlx5_ib_mw *mmw = to_mmw(mw); 1898 int err; 1899 1900 err = mlx5_core_destroy_mkey((to_mdev(mw->device))->mdev, 1901 &mmw->mmkey); 1902 if (!err) 1903 kfree(mmw); 1904 return err; 1905 } 1906 1907 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, 1908 struct ib_mr_status *mr_status) 1909 { 1910 struct mlx5_ib_mr *mmr = to_mmr(ibmr); 1911 int ret = 0; 1912 1913 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) { 1914 pr_err("Invalid status check mask\n"); 1915 ret = -EINVAL; 1916 goto done; 1917 } 1918 1919 mr_status->fail_status = 0; 1920 if (check_mask & IB_MR_CHECK_SIG_STATUS) { 1921 if (!mmr->sig) { 1922 ret = -EINVAL; 1923 pr_err("signature status check requested on a non-signature enabled MR\n"); 1924 goto done; 1925 } 1926 1927 mmr->sig->sig_status_checked = true; 1928 if (!mmr->sig->sig_err_exists) 1929 goto done; 1930 1931 if (ibmr->lkey == mmr->sig->err_item.key) 1932 memcpy(&mr_status->sig_err, &mmr->sig->err_item, 1933 sizeof(mr_status->sig_err)); 1934 else { 1935 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD; 1936 mr_status->sig_err.sig_err_offset = 0; 1937 mr_status->sig_err.key = mmr->sig->err_item.key; 1938 } 1939 1940 mmr->sig->sig_err_exists = false; 1941 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; 1942 } 1943 1944 done: 1945 return ret; 1946 } 1947 1948 static int 1949 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, 1950 struct scatterlist *sgl, 1951 unsigned short sg_nents, 1952 unsigned int *sg_offset_p) 1953 { 1954 struct scatterlist *sg = sgl; 1955 struct mlx5_klm *klms = mr->descs; 1956 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1957 u32 lkey = mr->ibmr.pd->local_dma_lkey; 1958 int i; 1959 1960 mr->ibmr.iova = sg_dma_address(sg) + sg_offset; 1961 mr->ibmr.length = 0; 1962 1963 for_each_sg(sgl, sg, sg_nents, i) { 1964 if (unlikely(i >= mr->max_descs)) 1965 break; 1966 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset); 1967 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset); 1968 klms[i].key = cpu_to_be32(lkey); 1969 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 1970 1971 sg_offset = 0; 1972 } 1973 mr->ndescs = i; 1974 1975 if (sg_offset_p) 1976 *sg_offset_p = sg_offset; 1977 1978 return i; 1979 } 1980 1981 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) 1982 { 1983 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1984 __be64 *descs; 1985 1986 if (unlikely(mr->ndescs == mr->max_descs)) 1987 return -ENOMEM; 1988 1989 descs = mr->descs; 1990 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 1991 1992 return 0; 1993 } 1994 1995 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1996 unsigned int *sg_offset) 1997 { 1998 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1999 int n; 2000 2001 mr->ndescs = 0; 2002 2003 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, 2004 mr->desc_size * mr->max_descs, 2005 DMA_TO_DEVICE); 2006 2007 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 2008 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset); 2009 else 2010 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, 2011 mlx5_set_page); 2012 2013 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map, 2014 mr->desc_size * mr->max_descs, 2015 DMA_TO_DEVICE); 2016 2017 return n; 2018 } 2019