1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 34 #include <linux/kref.h> 35 #include <linux/random.h> 36 #include <linux/debugfs.h> 37 #include <linux/export.h> 38 #include <linux/delay.h> 39 #include <rdma/ib_umem.h> 40 #include <rdma/ib_umem_odp.h> 41 #include <rdma/ib_verbs.h> 42 #include "mlx5_ib.h" 43 44 enum { 45 MAX_PENDING_REG_MR = 8, 46 }; 47 48 #define MLX5_UMR_ALIGN 2048 49 50 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 51 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 52 static int mr_cache_max_order(struct mlx5_ib_dev *dev); 53 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 54 static bool umr_can_modify_entity_size(struct mlx5_ib_dev *dev) 55 { 56 return !MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled); 57 } 58 59 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) 60 { 61 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled); 62 } 63 64 static bool use_umr(struct mlx5_ib_dev *dev, int order) 65 { 66 return order <= mr_cache_max_order(dev) && 67 umr_can_modify_entity_size(dev); 68 } 69 70 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 71 { 72 int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 73 74 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 75 /* Wait until all page fault handlers using the mr complete. */ 76 synchronize_srcu(&dev->mr_srcu); 77 #endif 78 79 return err; 80 } 81 82 static int order2idx(struct mlx5_ib_dev *dev, int order) 83 { 84 struct mlx5_mr_cache *cache = &dev->cache; 85 86 if (order < cache->ent[0].order) 87 return 0; 88 else 89 return order - cache->ent[0].order; 90 } 91 92 static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length) 93 { 94 return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >= 95 length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1)); 96 } 97 98 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 99 static void update_odp_mr(struct mlx5_ib_mr *mr) 100 { 101 if (mr->umem->is_odp) { 102 /* 103 * This barrier prevents the compiler from moving the 104 * setting of umem->odp_data->private to point to our 105 * MR, before reg_umr finished, to ensure that the MR 106 * initialization have finished before starting to 107 * handle invalidations. 108 */ 109 smp_wmb(); 110 to_ib_umem_odp(mr->umem)->private = mr; 111 /* 112 * Make sure we will see the new 113 * umem->odp_data->private value in the invalidation 114 * routines, before we can get page faults on the 115 * MR. Page faults can happen once we put the MR in 116 * the tree, below this line. Without the barrier, 117 * there can be a fault handling and an invalidation 118 * before umem->odp_data->private == mr is visible to 119 * the invalidation handler. 120 */ 121 smp_wmb(); 122 } 123 } 124 #endif 125 126 static void reg_mr_callback(int status, struct mlx5_async_work *context) 127 { 128 struct mlx5_ib_mr *mr = 129 container_of(context, struct mlx5_ib_mr, cb_work); 130 struct mlx5_ib_dev *dev = mr->dev; 131 struct mlx5_mr_cache *cache = &dev->cache; 132 int c = order2idx(dev, mr->order); 133 struct mlx5_cache_ent *ent = &cache->ent[c]; 134 u8 key; 135 unsigned long flags; 136 struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table; 137 int err; 138 139 spin_lock_irqsave(&ent->lock, flags); 140 ent->pending--; 141 spin_unlock_irqrestore(&ent->lock, flags); 142 if (status) { 143 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status); 144 kfree(mr); 145 dev->fill_delay = 1; 146 mod_timer(&dev->delay_timer, jiffies + HZ); 147 return; 148 } 149 150 mr->mmkey.type = MLX5_MKEY_MR; 151 spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags); 152 key = dev->mdev->priv.mkey_key++; 153 spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags); 154 mr->mmkey.key = mlx5_idx_to_mkey(MLX5_GET(create_mkey_out, mr->out, mkey_index)) | key; 155 156 cache->last_add = jiffies; 157 158 spin_lock_irqsave(&ent->lock, flags); 159 list_add_tail(&mr->list, &ent->head); 160 ent->cur++; 161 ent->size++; 162 spin_unlock_irqrestore(&ent->lock, flags); 163 164 write_lock_irqsave(&table->lock, flags); 165 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmkey.key), 166 &mr->mmkey); 167 if (err) 168 pr_err("Error inserting to mkey tree. 0x%x\n", -err); 169 write_unlock_irqrestore(&table->lock, flags); 170 171 if (!completion_done(&ent->compl)) 172 complete(&ent->compl); 173 } 174 175 static int add_keys(struct mlx5_ib_dev *dev, int c, int num) 176 { 177 struct mlx5_mr_cache *cache = &dev->cache; 178 struct mlx5_cache_ent *ent = &cache->ent[c]; 179 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 180 struct mlx5_ib_mr *mr; 181 void *mkc; 182 u32 *in; 183 int err = 0; 184 int i; 185 186 in = kzalloc(inlen, GFP_KERNEL); 187 if (!in) 188 return -ENOMEM; 189 190 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 191 for (i = 0; i < num; i++) { 192 if (ent->pending >= MAX_PENDING_REG_MR) { 193 err = -EAGAIN; 194 break; 195 } 196 197 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 198 if (!mr) { 199 err = -ENOMEM; 200 break; 201 } 202 mr->order = ent->order; 203 mr->allocated_from_cache = 1; 204 mr->dev = dev; 205 206 MLX5_SET(mkc, mkc, free, 1); 207 MLX5_SET(mkc, mkc, umr_en, 1); 208 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3); 209 MLX5_SET(mkc, mkc, access_mode_4_2, 210 (ent->access_mode >> 2) & 0x7); 211 212 MLX5_SET(mkc, mkc, qpn, 0xffffff); 213 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt); 214 MLX5_SET(mkc, mkc, log_page_size, ent->page); 215 216 spin_lock_irq(&ent->lock); 217 ent->pending++; 218 spin_unlock_irq(&ent->lock); 219 err = mlx5_core_create_mkey_cb(dev->mdev, &mr->mmkey, 220 &dev->async_ctx, in, inlen, 221 mr->out, sizeof(mr->out), 222 reg_mr_callback, &mr->cb_work); 223 if (err) { 224 spin_lock_irq(&ent->lock); 225 ent->pending--; 226 spin_unlock_irq(&ent->lock); 227 mlx5_ib_warn(dev, "create mkey failed %d\n", err); 228 kfree(mr); 229 break; 230 } 231 } 232 233 kfree(in); 234 return err; 235 } 236 237 static void remove_keys(struct mlx5_ib_dev *dev, int c, int num) 238 { 239 struct mlx5_mr_cache *cache = &dev->cache; 240 struct mlx5_cache_ent *ent = &cache->ent[c]; 241 struct mlx5_ib_mr *tmp_mr; 242 struct mlx5_ib_mr *mr; 243 LIST_HEAD(del_list); 244 int i; 245 246 for (i = 0; i < num; i++) { 247 spin_lock_irq(&ent->lock); 248 if (list_empty(&ent->head)) { 249 spin_unlock_irq(&ent->lock); 250 break; 251 } 252 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 253 list_move(&mr->list, &del_list); 254 ent->cur--; 255 ent->size--; 256 spin_unlock_irq(&ent->lock); 257 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 258 } 259 260 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 261 synchronize_srcu(&dev->mr_srcu); 262 #endif 263 264 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { 265 list_del(&mr->list); 266 kfree(mr); 267 } 268 } 269 270 static ssize_t size_write(struct file *filp, const char __user *buf, 271 size_t count, loff_t *pos) 272 { 273 struct mlx5_cache_ent *ent = filp->private_data; 274 struct mlx5_ib_dev *dev = ent->dev; 275 char lbuf[20] = {0}; 276 u32 var; 277 int err; 278 int c; 279 280 count = min(count, sizeof(lbuf) - 1); 281 if (copy_from_user(lbuf, buf, count)) 282 return -EFAULT; 283 284 c = order2idx(dev, ent->order); 285 286 if (sscanf(lbuf, "%u", &var) != 1) 287 return -EINVAL; 288 289 if (var < ent->limit) 290 return -EINVAL; 291 292 if (var > ent->size) { 293 do { 294 err = add_keys(dev, c, var - ent->size); 295 if (err && err != -EAGAIN) 296 return err; 297 298 usleep_range(3000, 5000); 299 } while (err); 300 } else if (var < ent->size) { 301 remove_keys(dev, c, ent->size - var); 302 } 303 304 return count; 305 } 306 307 static ssize_t size_read(struct file *filp, char __user *buf, size_t count, 308 loff_t *pos) 309 { 310 struct mlx5_cache_ent *ent = filp->private_data; 311 char lbuf[20]; 312 int err; 313 314 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size); 315 if (err < 0) 316 return err; 317 318 return simple_read_from_buffer(buf, count, pos, lbuf, err); 319 } 320 321 static const struct file_operations size_fops = { 322 .owner = THIS_MODULE, 323 .open = simple_open, 324 .write = size_write, 325 .read = size_read, 326 }; 327 328 static ssize_t limit_write(struct file *filp, const char __user *buf, 329 size_t count, loff_t *pos) 330 { 331 struct mlx5_cache_ent *ent = filp->private_data; 332 struct mlx5_ib_dev *dev = ent->dev; 333 char lbuf[20] = {0}; 334 u32 var; 335 int err; 336 int c; 337 338 count = min(count, sizeof(lbuf) - 1); 339 if (copy_from_user(lbuf, buf, count)) 340 return -EFAULT; 341 342 c = order2idx(dev, ent->order); 343 344 if (sscanf(lbuf, "%u", &var) != 1) 345 return -EINVAL; 346 347 if (var > ent->size) 348 return -EINVAL; 349 350 ent->limit = var; 351 352 if (ent->cur < ent->limit) { 353 err = add_keys(dev, c, 2 * ent->limit - ent->cur); 354 if (err) 355 return err; 356 } 357 358 return count; 359 } 360 361 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count, 362 loff_t *pos) 363 { 364 struct mlx5_cache_ent *ent = filp->private_data; 365 char lbuf[20]; 366 int err; 367 368 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); 369 if (err < 0) 370 return err; 371 372 return simple_read_from_buffer(buf, count, pos, lbuf, err); 373 } 374 375 static const struct file_operations limit_fops = { 376 .owner = THIS_MODULE, 377 .open = simple_open, 378 .write = limit_write, 379 .read = limit_read, 380 }; 381 382 static int someone_adding(struct mlx5_mr_cache *cache) 383 { 384 int i; 385 386 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 387 if (cache->ent[i].cur < cache->ent[i].limit) 388 return 1; 389 } 390 391 return 0; 392 } 393 394 static void __cache_work_func(struct mlx5_cache_ent *ent) 395 { 396 struct mlx5_ib_dev *dev = ent->dev; 397 struct mlx5_mr_cache *cache = &dev->cache; 398 int i = order2idx(dev, ent->order); 399 int err; 400 401 if (cache->stopped) 402 return; 403 404 ent = &dev->cache.ent[i]; 405 if (ent->cur < 2 * ent->limit && !dev->fill_delay) { 406 err = add_keys(dev, i, 1); 407 if (ent->cur < 2 * ent->limit) { 408 if (err == -EAGAIN) { 409 mlx5_ib_dbg(dev, "returned eagain, order %d\n", 410 i + 2); 411 queue_delayed_work(cache->wq, &ent->dwork, 412 msecs_to_jiffies(3)); 413 } else if (err) { 414 mlx5_ib_warn(dev, "command failed order %d, err %d\n", 415 i + 2, err); 416 queue_delayed_work(cache->wq, &ent->dwork, 417 msecs_to_jiffies(1000)); 418 } else { 419 queue_work(cache->wq, &ent->work); 420 } 421 } 422 } else if (ent->cur > 2 * ent->limit) { 423 /* 424 * The remove_keys() logic is performed as garbage collection 425 * task. Such task is intended to be run when no other active 426 * processes are running. 427 * 428 * The need_resched() will return TRUE if there are user tasks 429 * to be activated in near future. 430 * 431 * In such case, we don't execute remove_keys() and postpone 432 * the garbage collection work to try to run in next cycle, 433 * in order to free CPU resources to other tasks. 434 */ 435 if (!need_resched() && !someone_adding(cache) && 436 time_after(jiffies, cache->last_add + 300 * HZ)) { 437 remove_keys(dev, i, 1); 438 if (ent->cur > ent->limit) 439 queue_work(cache->wq, &ent->work); 440 } else { 441 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ); 442 } 443 } 444 } 445 446 static void delayed_cache_work_func(struct work_struct *work) 447 { 448 struct mlx5_cache_ent *ent; 449 450 ent = container_of(work, struct mlx5_cache_ent, dwork.work); 451 __cache_work_func(ent); 452 } 453 454 static void cache_work_func(struct work_struct *work) 455 { 456 struct mlx5_cache_ent *ent; 457 458 ent = container_of(work, struct mlx5_cache_ent, work); 459 __cache_work_func(ent); 460 } 461 462 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry) 463 { 464 struct mlx5_mr_cache *cache = &dev->cache; 465 struct mlx5_cache_ent *ent; 466 struct mlx5_ib_mr *mr; 467 int err; 468 469 if (entry < 0 || entry >= MAX_MR_CACHE_ENTRIES) { 470 mlx5_ib_err(dev, "cache entry %d is out of range\n", entry); 471 return NULL; 472 } 473 474 ent = &cache->ent[entry]; 475 while (1) { 476 spin_lock_irq(&ent->lock); 477 if (list_empty(&ent->head)) { 478 spin_unlock_irq(&ent->lock); 479 480 err = add_keys(dev, entry, 1); 481 if (err && err != -EAGAIN) 482 return ERR_PTR(err); 483 484 wait_for_completion(&ent->compl); 485 } else { 486 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, 487 list); 488 list_del(&mr->list); 489 ent->cur--; 490 spin_unlock_irq(&ent->lock); 491 if (ent->cur < ent->limit) 492 queue_work(cache->wq, &ent->work); 493 return mr; 494 } 495 } 496 } 497 498 static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order) 499 { 500 struct mlx5_mr_cache *cache = &dev->cache; 501 struct mlx5_ib_mr *mr = NULL; 502 struct mlx5_cache_ent *ent; 503 int last_umr_cache_entry; 504 int c; 505 int i; 506 507 c = order2idx(dev, order); 508 last_umr_cache_entry = order2idx(dev, mr_cache_max_order(dev)); 509 if (c < 0 || c > last_umr_cache_entry) { 510 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c); 511 return NULL; 512 } 513 514 for (i = c; i <= last_umr_cache_entry; i++) { 515 ent = &cache->ent[i]; 516 517 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i); 518 519 spin_lock_irq(&ent->lock); 520 if (!list_empty(&ent->head)) { 521 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, 522 list); 523 list_del(&mr->list); 524 ent->cur--; 525 spin_unlock_irq(&ent->lock); 526 if (ent->cur < ent->limit) 527 queue_work(cache->wq, &ent->work); 528 break; 529 } 530 spin_unlock_irq(&ent->lock); 531 532 queue_work(cache->wq, &ent->work); 533 } 534 535 if (!mr) 536 cache->ent[c].miss++; 537 538 return mr; 539 } 540 541 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 542 { 543 struct mlx5_mr_cache *cache = &dev->cache; 544 struct mlx5_cache_ent *ent; 545 int shrink = 0; 546 int c; 547 548 if (!mr->allocated_from_cache) 549 return; 550 551 c = order2idx(dev, mr->order); 552 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) { 553 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c); 554 return; 555 } 556 557 if (unreg_umr(dev, mr)) 558 return; 559 560 ent = &cache->ent[c]; 561 spin_lock_irq(&ent->lock); 562 list_add_tail(&mr->list, &ent->head); 563 ent->cur++; 564 if (ent->cur > 2 * ent->limit) 565 shrink = 1; 566 spin_unlock_irq(&ent->lock); 567 568 if (shrink) 569 queue_work(cache->wq, &ent->work); 570 } 571 572 static void clean_keys(struct mlx5_ib_dev *dev, int c) 573 { 574 struct mlx5_mr_cache *cache = &dev->cache; 575 struct mlx5_cache_ent *ent = &cache->ent[c]; 576 struct mlx5_ib_mr *tmp_mr; 577 struct mlx5_ib_mr *mr; 578 LIST_HEAD(del_list); 579 580 cancel_delayed_work(&ent->dwork); 581 while (1) { 582 spin_lock_irq(&ent->lock); 583 if (list_empty(&ent->head)) { 584 spin_unlock_irq(&ent->lock); 585 break; 586 } 587 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 588 list_move(&mr->list, &del_list); 589 ent->cur--; 590 ent->size--; 591 spin_unlock_irq(&ent->lock); 592 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 593 } 594 595 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 596 synchronize_srcu(&dev->mr_srcu); 597 #endif 598 599 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { 600 list_del(&mr->list); 601 kfree(mr); 602 } 603 } 604 605 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) 606 { 607 if (!mlx5_debugfs_root || dev->rep) 608 return; 609 610 debugfs_remove_recursive(dev->cache.root); 611 dev->cache.root = NULL; 612 } 613 614 static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev) 615 { 616 struct mlx5_mr_cache *cache = &dev->cache; 617 struct mlx5_cache_ent *ent; 618 int i; 619 620 if (!mlx5_debugfs_root || dev->rep) 621 return 0; 622 623 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root); 624 if (!cache->root) 625 return -ENOMEM; 626 627 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 628 ent = &cache->ent[i]; 629 sprintf(ent->name, "%d", ent->order); 630 ent->dir = debugfs_create_dir(ent->name, cache->root); 631 if (!ent->dir) 632 goto err; 633 634 ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent, 635 &size_fops); 636 if (!ent->fsize) 637 goto err; 638 639 ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent, 640 &limit_fops); 641 if (!ent->flimit) 642 goto err; 643 644 ent->fcur = debugfs_create_u32("cur", 0400, ent->dir, 645 &ent->cur); 646 if (!ent->fcur) 647 goto err; 648 649 ent->fmiss = debugfs_create_u32("miss", 0600, ent->dir, 650 &ent->miss); 651 if (!ent->fmiss) 652 goto err; 653 } 654 655 return 0; 656 err: 657 mlx5_mr_cache_debugfs_cleanup(dev); 658 659 return -ENOMEM; 660 } 661 662 static void delay_time_func(struct timer_list *t) 663 { 664 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer); 665 666 dev->fill_delay = 0; 667 } 668 669 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) 670 { 671 struct mlx5_mr_cache *cache = &dev->cache; 672 struct mlx5_cache_ent *ent; 673 int err; 674 int i; 675 676 mutex_init(&dev->slow_path_mutex); 677 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM); 678 if (!cache->wq) { 679 mlx5_ib_warn(dev, "failed to create work queue\n"); 680 return -ENOMEM; 681 } 682 683 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx); 684 timer_setup(&dev->delay_timer, delay_time_func, 0); 685 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 686 ent = &cache->ent[i]; 687 INIT_LIST_HEAD(&ent->head); 688 spin_lock_init(&ent->lock); 689 ent->order = i + 2; 690 ent->dev = dev; 691 ent->limit = 0; 692 693 init_completion(&ent->compl); 694 INIT_WORK(&ent->work, cache_work_func); 695 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 696 697 if (i > MR_CACHE_LAST_STD_ENTRY) { 698 mlx5_odp_init_mr_cache_entry(ent); 699 continue; 700 } 701 702 if (ent->order > mr_cache_max_order(dev)) 703 continue; 704 705 ent->page = PAGE_SHIFT; 706 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) / 707 MLX5_IB_UMR_OCTOWORD; 708 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 709 if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) && 710 !dev->rep && 711 mlx5_core_is_pf(dev->mdev)) 712 ent->limit = dev->mdev->profile->mr_cache[i].limit; 713 else 714 ent->limit = 0; 715 queue_work(cache->wq, &ent->work); 716 } 717 718 err = mlx5_mr_cache_debugfs_init(dev); 719 if (err) 720 mlx5_ib_warn(dev, "cache debugfs failure\n"); 721 722 /* 723 * We don't want to fail driver if debugfs failed to initialize, 724 * so we are not forwarding error to the user. 725 */ 726 727 return 0; 728 } 729 730 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) 731 { 732 int i; 733 734 if (!dev->cache.wq) 735 return 0; 736 737 dev->cache.stopped = 1; 738 flush_workqueue(dev->cache.wq); 739 740 mlx5_mr_cache_debugfs_cleanup(dev); 741 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx); 742 743 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) 744 clean_keys(dev, i); 745 746 destroy_workqueue(dev->cache.wq); 747 del_timer_sync(&dev->delay_timer); 748 749 return 0; 750 } 751 752 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) 753 { 754 struct mlx5_ib_dev *dev = to_mdev(pd->device); 755 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 756 struct mlx5_core_dev *mdev = dev->mdev; 757 struct mlx5_ib_mr *mr; 758 void *mkc; 759 u32 *in; 760 int err; 761 762 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 763 if (!mr) 764 return ERR_PTR(-ENOMEM); 765 766 in = kzalloc(inlen, GFP_KERNEL); 767 if (!in) { 768 err = -ENOMEM; 769 goto err_free; 770 } 771 772 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 773 774 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 775 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 776 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 777 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 778 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 779 MLX5_SET(mkc, mkc, lr, 1); 780 781 MLX5_SET(mkc, mkc, length64, 1); 782 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 783 MLX5_SET(mkc, mkc, qpn, 0xffffff); 784 MLX5_SET64(mkc, mkc, start_addr, 0); 785 786 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen); 787 if (err) 788 goto err_in; 789 790 kfree(in); 791 mr->mmkey.type = MLX5_MKEY_MR; 792 mr->ibmr.lkey = mr->mmkey.key; 793 mr->ibmr.rkey = mr->mmkey.key; 794 mr->umem = NULL; 795 796 return &mr->ibmr; 797 798 err_in: 799 kfree(in); 800 801 err_free: 802 kfree(mr); 803 804 return ERR_PTR(err); 805 } 806 807 static int get_octo_len(u64 addr, u64 len, int page_shift) 808 { 809 u64 page_size = 1ULL << page_shift; 810 u64 offset; 811 int npages; 812 813 offset = addr & (page_size - 1); 814 npages = ALIGN(len + offset, page_size) >> page_shift; 815 return (npages + 1) / 2; 816 } 817 818 static int mr_cache_max_order(struct mlx5_ib_dev *dev) 819 { 820 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 821 return MR_CACHE_LAST_STD_ENTRY + 2; 822 return MLX5_MAX_UMR_SHIFT; 823 } 824 825 static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length, 826 int access_flags, struct ib_umem **umem, 827 int *npages, int *page_shift, int *ncont, 828 int *order) 829 { 830 struct mlx5_ib_dev *dev = to_mdev(pd->device); 831 struct ib_umem *u; 832 int err; 833 834 *umem = NULL; 835 836 u = ib_umem_get(pd->uobject->context, start, length, access_flags, 0); 837 err = PTR_ERR_OR_ZERO(u); 838 if (err) { 839 mlx5_ib_dbg(dev, "umem get failed (%d)\n", err); 840 return err; 841 } 842 843 mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages, 844 page_shift, ncont, order); 845 if (!*npages) { 846 mlx5_ib_warn(dev, "avoid zero region\n"); 847 ib_umem_release(u); 848 return -EINVAL; 849 } 850 851 *umem = u; 852 853 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n", 854 *npages, *ncont, *order, *page_shift); 855 856 return 0; 857 } 858 859 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc) 860 { 861 struct mlx5_ib_umr_context *context = 862 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe); 863 864 context->status = wc->status; 865 complete(&context->done); 866 } 867 868 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context) 869 { 870 context->cqe.done = mlx5_ib_umr_done; 871 context->status = -1; 872 init_completion(&context->done); 873 } 874 875 static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev, 876 struct mlx5_umr_wr *umrwr) 877 { 878 struct umr_common *umrc = &dev->umrc; 879 const struct ib_send_wr *bad; 880 int err; 881 struct mlx5_ib_umr_context umr_context; 882 883 mlx5_ib_init_umr_context(&umr_context); 884 umrwr->wr.wr_cqe = &umr_context.cqe; 885 886 down(&umrc->sem); 887 err = ib_post_send(umrc->qp, &umrwr->wr, &bad); 888 if (err) { 889 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err); 890 } else { 891 wait_for_completion(&umr_context.done); 892 if (umr_context.status != IB_WC_SUCCESS) { 893 mlx5_ib_warn(dev, "reg umr failed (%u)\n", 894 umr_context.status); 895 err = -EFAULT; 896 } 897 } 898 up(&umrc->sem); 899 return err; 900 } 901 902 static struct mlx5_ib_mr *alloc_mr_from_cache( 903 struct ib_pd *pd, struct ib_umem *umem, 904 u64 virt_addr, u64 len, int npages, 905 int page_shift, int order, int access_flags) 906 { 907 struct mlx5_ib_dev *dev = to_mdev(pd->device); 908 struct mlx5_ib_mr *mr; 909 int err = 0; 910 int i; 911 912 for (i = 0; i < 1; i++) { 913 mr = alloc_cached_mr(dev, order); 914 if (mr) 915 break; 916 917 err = add_keys(dev, order2idx(dev, order), 1); 918 if (err && err != -EAGAIN) { 919 mlx5_ib_warn(dev, "add_keys failed, err %d\n", err); 920 break; 921 } 922 } 923 924 if (!mr) 925 return ERR_PTR(-EAGAIN); 926 927 mr->ibmr.pd = pd; 928 mr->umem = umem; 929 mr->access_flags = access_flags; 930 mr->desc_size = sizeof(struct mlx5_mtt); 931 mr->mmkey.iova = virt_addr; 932 mr->mmkey.size = len; 933 mr->mmkey.pd = to_mpd(pd)->pdn; 934 935 return mr; 936 } 937 938 static inline int populate_xlt(struct mlx5_ib_mr *mr, int idx, int npages, 939 void *xlt, int page_shift, size_t size, 940 int flags) 941 { 942 struct mlx5_ib_dev *dev = mr->dev; 943 struct ib_umem *umem = mr->umem; 944 945 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 946 if (!umr_can_use_indirect_mkey(dev)) 947 return -EPERM; 948 mlx5_odp_populate_klm(xlt, idx, npages, mr, flags); 949 return npages; 950 } 951 952 npages = min_t(size_t, npages, ib_umem_num_pages(umem) - idx); 953 954 if (!(flags & MLX5_IB_UPD_XLT_ZAP)) { 955 __mlx5_ib_populate_pas(dev, umem, page_shift, 956 idx, npages, xlt, 957 MLX5_IB_MTT_PRESENT); 958 /* Clear padding after the pages 959 * brought from the umem. 960 */ 961 memset(xlt + (npages * sizeof(struct mlx5_mtt)), 0, 962 size - npages * sizeof(struct mlx5_mtt)); 963 } 964 965 return npages; 966 } 967 968 #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \ 969 MLX5_UMR_MTT_ALIGNMENT) 970 #define MLX5_SPARE_UMR_CHUNK 0x10000 971 972 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, 973 int page_shift, int flags) 974 { 975 struct mlx5_ib_dev *dev = mr->dev; 976 struct device *ddev = dev->ib_dev.dev.parent; 977 int size; 978 void *xlt; 979 dma_addr_t dma; 980 struct mlx5_umr_wr wr; 981 struct ib_sge sg; 982 int err = 0; 983 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) 984 ? sizeof(struct mlx5_klm) 985 : sizeof(struct mlx5_mtt); 986 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size; 987 const int page_mask = page_align - 1; 988 size_t pages_mapped = 0; 989 size_t pages_to_map = 0; 990 size_t pages_iter = 0; 991 gfp_t gfp; 992 bool use_emergency_page = false; 993 994 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) && 995 !umr_can_use_indirect_mkey(dev)) 996 return -EPERM; 997 998 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes, 999 * so we need to align the offset and length accordingly 1000 */ 1001 if (idx & page_mask) { 1002 npages += idx & page_mask; 1003 idx &= ~page_mask; 1004 } 1005 1006 gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL; 1007 gfp |= __GFP_ZERO | __GFP_NOWARN; 1008 1009 pages_to_map = ALIGN(npages, page_align); 1010 size = desc_size * pages_to_map; 1011 size = min_t(int, size, MLX5_MAX_UMR_CHUNK); 1012 1013 xlt = (void *)__get_free_pages(gfp, get_order(size)); 1014 if (!xlt && size > MLX5_SPARE_UMR_CHUNK) { 1015 mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n", 1016 size, get_order(size), MLX5_SPARE_UMR_CHUNK); 1017 1018 size = MLX5_SPARE_UMR_CHUNK; 1019 xlt = (void *)__get_free_pages(gfp, get_order(size)); 1020 } 1021 1022 if (!xlt) { 1023 mlx5_ib_warn(dev, "Using XLT emergency buffer\n"); 1024 xlt = (void *)mlx5_ib_get_xlt_emergency_page(); 1025 size = PAGE_SIZE; 1026 memset(xlt, 0, size); 1027 use_emergency_page = true; 1028 } 1029 pages_iter = size / desc_size; 1030 dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE); 1031 if (dma_mapping_error(ddev, dma)) { 1032 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n"); 1033 err = -ENOMEM; 1034 goto free_xlt; 1035 } 1036 1037 sg.addr = dma; 1038 sg.lkey = dev->umrc.pd->local_dma_lkey; 1039 1040 memset(&wr, 0, sizeof(wr)); 1041 wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT; 1042 if (!(flags & MLX5_IB_UPD_XLT_ENABLE)) 1043 wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1044 wr.wr.sg_list = &sg; 1045 wr.wr.num_sge = 1; 1046 wr.wr.opcode = MLX5_IB_WR_UMR; 1047 1048 wr.pd = mr->ibmr.pd; 1049 wr.mkey = mr->mmkey.key; 1050 wr.length = mr->mmkey.size; 1051 wr.virt_addr = mr->mmkey.iova; 1052 wr.access_flags = mr->access_flags; 1053 wr.page_shift = page_shift; 1054 1055 for (pages_mapped = 0; 1056 pages_mapped < pages_to_map && !err; 1057 pages_mapped += pages_iter, idx += pages_iter) { 1058 npages = min_t(int, pages_iter, pages_to_map - pages_mapped); 1059 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE); 1060 npages = populate_xlt(mr, idx, npages, xlt, 1061 page_shift, size, flags); 1062 1063 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE); 1064 1065 sg.length = ALIGN(npages * desc_size, 1066 MLX5_UMR_MTT_ALIGNMENT); 1067 1068 if (pages_mapped + pages_iter >= pages_to_map) { 1069 if (flags & MLX5_IB_UPD_XLT_ENABLE) 1070 wr.wr.send_flags |= 1071 MLX5_IB_SEND_UMR_ENABLE_MR | 1072 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS | 1073 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1074 if (flags & MLX5_IB_UPD_XLT_PD || 1075 flags & MLX5_IB_UPD_XLT_ACCESS) 1076 wr.wr.send_flags |= 1077 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1078 if (flags & MLX5_IB_UPD_XLT_ADDR) 1079 wr.wr.send_flags |= 1080 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1081 } 1082 1083 wr.offset = idx * desc_size; 1084 wr.xlt_size = sg.length; 1085 1086 err = mlx5_ib_post_send_wait(dev, &wr); 1087 } 1088 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE); 1089 1090 free_xlt: 1091 if (use_emergency_page) 1092 mlx5_ib_put_xlt_emergency_page(); 1093 else 1094 free_pages((unsigned long)xlt, get_order(size)); 1095 1096 return err; 1097 } 1098 1099 /* 1100 * If ibmr is NULL it will be allocated by reg_create. 1101 * Else, the given ibmr will be used. 1102 */ 1103 static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, 1104 u64 virt_addr, u64 length, 1105 struct ib_umem *umem, int npages, 1106 int page_shift, int access_flags, 1107 bool populate) 1108 { 1109 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1110 struct mlx5_ib_mr *mr; 1111 __be64 *pas; 1112 void *mkc; 1113 int inlen; 1114 u32 *in; 1115 int err; 1116 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 1117 1118 mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL); 1119 if (!mr) 1120 return ERR_PTR(-ENOMEM); 1121 1122 mr->ibmr.pd = pd; 1123 mr->access_flags = access_flags; 1124 1125 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1126 if (populate) 1127 inlen += sizeof(*pas) * roundup(npages, 2); 1128 in = kvzalloc(inlen, GFP_KERNEL); 1129 if (!in) { 1130 err = -ENOMEM; 1131 goto err_1; 1132 } 1133 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 1134 if (populate && !(access_flags & IB_ACCESS_ON_DEMAND)) 1135 mlx5_ib_populate_pas(dev, umem, page_shift, pas, 1136 pg_cap ? MLX5_IB_MTT_PRESENT : 0); 1137 1138 /* The pg_access bit allows setting the access flags 1139 * in the page list submitted with the command. */ 1140 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap)); 1141 1142 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1143 MLX5_SET(mkc, mkc, free, !populate); 1144 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 1145 MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); 1146 MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE)); 1147 MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ)); 1148 MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE)); 1149 MLX5_SET(mkc, mkc, lr, 1); 1150 MLX5_SET(mkc, mkc, umr_en, 1); 1151 1152 MLX5_SET64(mkc, mkc, start_addr, virt_addr); 1153 MLX5_SET64(mkc, mkc, len, length); 1154 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1155 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 1156 MLX5_SET(mkc, mkc, translations_octword_size, 1157 get_octo_len(virt_addr, length, page_shift)); 1158 MLX5_SET(mkc, mkc, log_page_size, page_shift); 1159 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1160 if (populate) { 1161 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 1162 get_octo_len(virt_addr, length, page_shift)); 1163 } 1164 1165 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); 1166 if (err) { 1167 mlx5_ib_warn(dev, "create mkey failed\n"); 1168 goto err_2; 1169 } 1170 mr->mmkey.type = MLX5_MKEY_MR; 1171 mr->desc_size = sizeof(struct mlx5_mtt); 1172 mr->dev = dev; 1173 kvfree(in); 1174 1175 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key); 1176 1177 return mr; 1178 1179 err_2: 1180 kvfree(in); 1181 1182 err_1: 1183 if (!ibmr) 1184 kfree(mr); 1185 1186 return ERR_PTR(err); 1187 } 1188 1189 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 1190 int npages, u64 length, int access_flags) 1191 { 1192 mr->npages = npages; 1193 atomic_add(npages, &dev->mdev->priv.reg_pages); 1194 mr->ibmr.lkey = mr->mmkey.key; 1195 mr->ibmr.rkey = mr->mmkey.key; 1196 mr->ibmr.length = length; 1197 mr->access_flags = access_flags; 1198 } 1199 1200 static struct ib_mr *mlx5_ib_get_memic_mr(struct ib_pd *pd, u64 memic_addr, 1201 u64 length, int acc) 1202 { 1203 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1204 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1205 struct mlx5_core_dev *mdev = dev->mdev; 1206 struct mlx5_ib_mr *mr; 1207 void *mkc; 1208 u32 *in; 1209 int err; 1210 1211 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1212 if (!mr) 1213 return ERR_PTR(-ENOMEM); 1214 1215 in = kzalloc(inlen, GFP_KERNEL); 1216 if (!in) { 1217 err = -ENOMEM; 1218 goto err_free; 1219 } 1220 1221 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1222 1223 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MEMIC & 0x3); 1224 MLX5_SET(mkc, mkc, access_mode_4_2, 1225 (MLX5_MKC_ACCESS_MODE_MEMIC >> 2) & 0x7); 1226 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 1227 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 1228 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 1229 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 1230 MLX5_SET(mkc, mkc, lr, 1); 1231 1232 MLX5_SET64(mkc, mkc, len, length); 1233 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1234 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1235 MLX5_SET64(mkc, mkc, start_addr, 1236 memic_addr - pci_resource_start(dev->mdev->pdev, 0)); 1237 1238 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen); 1239 if (err) 1240 goto err_in; 1241 1242 kfree(in); 1243 1244 mr->umem = NULL; 1245 set_mr_fields(dev, mr, 0, length, acc); 1246 1247 return &mr->ibmr; 1248 1249 err_in: 1250 kfree(in); 1251 1252 err_free: 1253 kfree(mr); 1254 1255 return ERR_PTR(err); 1256 } 1257 1258 int mlx5_ib_advise_mr(struct ib_pd *pd, 1259 enum ib_uverbs_advise_mr_advice advice, 1260 u32 flags, 1261 struct ib_sge *sg_list, 1262 u32 num_sge, 1263 struct uverbs_attr_bundle *attrs) 1264 { 1265 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && 1266 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE) 1267 return -EOPNOTSUPP; 1268 1269 return mlx5_ib_advise_mr_prefetch(pd, advice, flags, 1270 sg_list, num_sge); 1271 } 1272 1273 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, 1274 struct ib_dm_mr_attr *attr, 1275 struct uverbs_attr_bundle *attrs) 1276 { 1277 struct mlx5_ib_dm *mdm = to_mdm(dm); 1278 u64 memic_addr; 1279 1280 if (attr->access_flags & ~MLX5_IB_DM_ALLOWED_ACCESS) 1281 return ERR_PTR(-EINVAL); 1282 1283 memic_addr = mdm->dev_addr + attr->offset; 1284 1285 return mlx5_ib_get_memic_mr(pd, memic_addr, attr->length, 1286 attr->access_flags); 1287 } 1288 1289 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 1290 u64 virt_addr, int access_flags, 1291 struct ib_udata *udata) 1292 { 1293 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1294 struct mlx5_ib_mr *mr = NULL; 1295 bool populate_mtts = false; 1296 struct ib_umem *umem; 1297 int page_shift; 1298 int npages; 1299 int ncont; 1300 int order; 1301 int err; 1302 1303 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) 1304 return ERR_PTR(-EOPNOTSUPP); 1305 1306 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1307 start, virt_addr, length, access_flags); 1308 1309 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1310 if (!start && length == U64_MAX) { 1311 if (!(access_flags & IB_ACCESS_ON_DEMAND) || 1312 !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1313 return ERR_PTR(-EINVAL); 1314 1315 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), access_flags); 1316 if (IS_ERR(mr)) 1317 return ERR_CAST(mr); 1318 return &mr->ibmr; 1319 } 1320 #endif 1321 1322 err = mr_umem_get(pd, start, length, access_flags, &umem, &npages, 1323 &page_shift, &ncont, &order); 1324 1325 if (err < 0) 1326 return ERR_PTR(err); 1327 1328 if (use_umr(dev, order)) { 1329 mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont, 1330 page_shift, order, access_flags); 1331 if (PTR_ERR(mr) == -EAGAIN) { 1332 mlx5_ib_dbg(dev, "cache empty for order %d\n", order); 1333 mr = NULL; 1334 } 1335 populate_mtts = false; 1336 } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) { 1337 if (access_flags & IB_ACCESS_ON_DEMAND) { 1338 err = -EINVAL; 1339 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n"); 1340 goto error; 1341 } 1342 populate_mtts = true; 1343 } 1344 1345 if (!mr) { 1346 if (!umr_can_modify_entity_size(dev)) 1347 populate_mtts = true; 1348 mutex_lock(&dev->slow_path_mutex); 1349 mr = reg_create(NULL, pd, virt_addr, length, umem, ncont, 1350 page_shift, access_flags, populate_mtts); 1351 mutex_unlock(&dev->slow_path_mutex); 1352 } 1353 1354 if (IS_ERR(mr)) { 1355 err = PTR_ERR(mr); 1356 goto error; 1357 } 1358 1359 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); 1360 1361 mr->umem = umem; 1362 set_mr_fields(dev, mr, npages, length, access_flags); 1363 1364 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1365 update_odp_mr(mr); 1366 #endif 1367 1368 if (!populate_mtts) { 1369 int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE; 1370 1371 if (access_flags & IB_ACCESS_ON_DEMAND) 1372 update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP; 1373 1374 err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift, 1375 update_xlt_flags); 1376 1377 if (err) { 1378 dereg_mr(dev, mr); 1379 return ERR_PTR(err); 1380 } 1381 } 1382 1383 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1384 mr->live = 1; 1385 #endif 1386 return &mr->ibmr; 1387 error: 1388 ib_umem_release(umem); 1389 return ERR_PTR(err); 1390 } 1391 1392 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1393 { 1394 struct mlx5_core_dev *mdev = dev->mdev; 1395 struct mlx5_umr_wr umrwr = {}; 1396 1397 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 1398 return 0; 1399 1400 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR | 1401 MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1402 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1403 umrwr.mkey = mr->mmkey.key; 1404 1405 return mlx5_ib_post_send_wait(dev, &umrwr); 1406 } 1407 1408 static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1409 int access_flags, int flags) 1410 { 1411 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1412 struct mlx5_umr_wr umrwr = {}; 1413 int err; 1414 1415 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1416 1417 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1418 umrwr.mkey = mr->mmkey.key; 1419 1420 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) { 1421 umrwr.pd = pd; 1422 umrwr.access_flags = access_flags; 1423 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1424 } 1425 1426 err = mlx5_ib_post_send_wait(dev, &umrwr); 1427 1428 return err; 1429 } 1430 1431 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 1432 u64 length, u64 virt_addr, int new_access_flags, 1433 struct ib_pd *new_pd, struct ib_udata *udata) 1434 { 1435 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device); 1436 struct mlx5_ib_mr *mr = to_mmr(ib_mr); 1437 struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd; 1438 int access_flags = flags & IB_MR_REREG_ACCESS ? 1439 new_access_flags : 1440 mr->access_flags; 1441 int page_shift = 0; 1442 int upd_flags = 0; 1443 int npages = 0; 1444 int ncont = 0; 1445 int order = 0; 1446 u64 addr, len; 1447 int err; 1448 1449 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1450 start, virt_addr, length, access_flags); 1451 1452 atomic_sub(mr->npages, &dev->mdev->priv.reg_pages); 1453 1454 if (!mr->umem) 1455 return -EINVAL; 1456 1457 if (flags & IB_MR_REREG_TRANS) { 1458 addr = virt_addr; 1459 len = length; 1460 } else { 1461 addr = mr->umem->address; 1462 len = mr->umem->length; 1463 } 1464 1465 if (flags != IB_MR_REREG_PD) { 1466 /* 1467 * Replace umem. This needs to be done whether or not UMR is 1468 * used. 1469 */ 1470 flags |= IB_MR_REREG_TRANS; 1471 ib_umem_release(mr->umem); 1472 mr->umem = NULL; 1473 err = mr_umem_get(pd, addr, len, access_flags, &mr->umem, 1474 &npages, &page_shift, &ncont, &order); 1475 if (err) 1476 goto err; 1477 } 1478 1479 if (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len)) { 1480 /* 1481 * UMR can't be used - MKey needs to be replaced. 1482 */ 1483 if (mr->allocated_from_cache) 1484 err = unreg_umr(dev, mr); 1485 else 1486 err = destroy_mkey(dev, mr); 1487 if (err) 1488 goto err; 1489 1490 mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont, 1491 page_shift, access_flags, true); 1492 1493 if (IS_ERR(mr)) { 1494 err = PTR_ERR(mr); 1495 mr = to_mmr(ib_mr); 1496 goto err; 1497 } 1498 1499 mr->allocated_from_cache = 0; 1500 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1501 mr->live = 1; 1502 #endif 1503 } else { 1504 /* 1505 * Send a UMR WQE 1506 */ 1507 mr->ibmr.pd = pd; 1508 mr->access_flags = access_flags; 1509 mr->mmkey.iova = addr; 1510 mr->mmkey.size = len; 1511 mr->mmkey.pd = to_mpd(pd)->pdn; 1512 1513 if (flags & IB_MR_REREG_TRANS) { 1514 upd_flags = MLX5_IB_UPD_XLT_ADDR; 1515 if (flags & IB_MR_REREG_PD) 1516 upd_flags |= MLX5_IB_UPD_XLT_PD; 1517 if (flags & IB_MR_REREG_ACCESS) 1518 upd_flags |= MLX5_IB_UPD_XLT_ACCESS; 1519 err = mlx5_ib_update_xlt(mr, 0, npages, page_shift, 1520 upd_flags); 1521 } else { 1522 err = rereg_umr(pd, mr, access_flags, flags); 1523 } 1524 1525 if (err) 1526 goto err; 1527 } 1528 1529 set_mr_fields(dev, mr, npages, len, access_flags); 1530 1531 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1532 update_odp_mr(mr); 1533 #endif 1534 return 0; 1535 1536 err: 1537 if (mr->umem) { 1538 ib_umem_release(mr->umem); 1539 mr->umem = NULL; 1540 } 1541 clean_mr(dev, mr); 1542 return err; 1543 } 1544 1545 static int 1546 mlx5_alloc_priv_descs(struct ib_device *device, 1547 struct mlx5_ib_mr *mr, 1548 int ndescs, 1549 int desc_size) 1550 { 1551 int size = ndescs * desc_size; 1552 int add_size; 1553 int ret; 1554 1555 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0); 1556 1557 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL); 1558 if (!mr->descs_alloc) 1559 return -ENOMEM; 1560 1561 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN); 1562 1563 mr->desc_map = dma_map_single(device->dev.parent, mr->descs, 1564 size, DMA_TO_DEVICE); 1565 if (dma_mapping_error(device->dev.parent, mr->desc_map)) { 1566 ret = -ENOMEM; 1567 goto err; 1568 } 1569 1570 return 0; 1571 err: 1572 kfree(mr->descs_alloc); 1573 1574 return ret; 1575 } 1576 1577 static void 1578 mlx5_free_priv_descs(struct mlx5_ib_mr *mr) 1579 { 1580 if (mr->descs) { 1581 struct ib_device *device = mr->ibmr.device; 1582 int size = mr->max_descs * mr->desc_size; 1583 1584 dma_unmap_single(device->dev.parent, mr->desc_map, 1585 size, DMA_TO_DEVICE); 1586 kfree(mr->descs_alloc); 1587 mr->descs = NULL; 1588 } 1589 } 1590 1591 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1592 { 1593 int allocated_from_cache = mr->allocated_from_cache; 1594 1595 if (mr->sig) { 1596 if (mlx5_core_destroy_psv(dev->mdev, 1597 mr->sig->psv_memory.psv_idx)) 1598 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1599 mr->sig->psv_memory.psv_idx); 1600 if (mlx5_core_destroy_psv(dev->mdev, 1601 mr->sig->psv_wire.psv_idx)) 1602 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1603 mr->sig->psv_wire.psv_idx); 1604 kfree(mr->sig); 1605 mr->sig = NULL; 1606 } 1607 1608 mlx5_free_priv_descs(mr); 1609 1610 if (!allocated_from_cache) 1611 destroy_mkey(dev, mr); 1612 } 1613 1614 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1615 { 1616 int npages = mr->npages; 1617 struct ib_umem *umem = mr->umem; 1618 1619 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1620 if (umem && umem->is_odp) { 1621 struct ib_umem_odp *umem_odp = to_ib_umem_odp(umem); 1622 1623 /* Prevent new page faults from succeeding */ 1624 mr->live = 0; 1625 /* Wait for all running page-fault handlers to finish. */ 1626 synchronize_srcu(&dev->mr_srcu); 1627 /* Destroy all page mappings */ 1628 if (umem_odp->page_list) 1629 mlx5_ib_invalidate_range(umem_odp, ib_umem_start(umem), 1630 ib_umem_end(umem)); 1631 else 1632 mlx5_ib_free_implicit_mr(mr); 1633 /* 1634 * We kill the umem before the MR for ODP, 1635 * so that there will not be any invalidations in 1636 * flight, looking at the *mr struct. 1637 */ 1638 ib_umem_release(umem); 1639 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1640 1641 /* Avoid double-freeing the umem. */ 1642 umem = NULL; 1643 } 1644 #endif 1645 clean_mr(dev, mr); 1646 1647 /* 1648 * We should unregister the DMA address from the HCA before 1649 * remove the DMA mapping. 1650 */ 1651 mlx5_mr_cache_free(dev, mr); 1652 if (umem) { 1653 ib_umem_release(umem); 1654 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1655 } 1656 if (!mr->allocated_from_cache) 1657 kfree(mr); 1658 } 1659 1660 int mlx5_ib_dereg_mr(struct ib_mr *ibmr) 1661 { 1662 dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr)); 1663 return 0; 1664 } 1665 1666 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, 1667 enum ib_mr_type mr_type, 1668 u32 max_num_sg) 1669 { 1670 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1671 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1672 int ndescs = ALIGN(max_num_sg, 4); 1673 struct mlx5_ib_mr *mr; 1674 void *mkc; 1675 u32 *in; 1676 int err; 1677 1678 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1679 if (!mr) 1680 return ERR_PTR(-ENOMEM); 1681 1682 in = kzalloc(inlen, GFP_KERNEL); 1683 if (!in) { 1684 err = -ENOMEM; 1685 goto err_free; 1686 } 1687 1688 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1689 MLX5_SET(mkc, mkc, free, 1); 1690 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 1691 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1692 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1693 1694 if (mr_type == IB_MR_TYPE_MEM_REG) { 1695 mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1696 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 1697 err = mlx5_alloc_priv_descs(pd->device, mr, 1698 ndescs, sizeof(struct mlx5_mtt)); 1699 if (err) 1700 goto err_free_in; 1701 1702 mr->desc_size = sizeof(struct mlx5_mtt); 1703 mr->max_descs = ndescs; 1704 } else if (mr_type == IB_MR_TYPE_SG_GAPS) { 1705 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; 1706 1707 err = mlx5_alloc_priv_descs(pd->device, mr, 1708 ndescs, sizeof(struct mlx5_klm)); 1709 if (err) 1710 goto err_free_in; 1711 mr->desc_size = sizeof(struct mlx5_klm); 1712 mr->max_descs = ndescs; 1713 } else if (mr_type == IB_MR_TYPE_SIGNATURE) { 1714 u32 psv_index[2]; 1715 1716 MLX5_SET(mkc, mkc, bsf_en, 1); 1717 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); 1718 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); 1719 if (!mr->sig) { 1720 err = -ENOMEM; 1721 goto err_free_in; 1722 } 1723 1724 /* create mem & wire PSVs */ 1725 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 1726 2, psv_index); 1727 if (err) 1728 goto err_free_sig; 1729 1730 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; 1731 mr->sig->psv_memory.psv_idx = psv_index[0]; 1732 mr->sig->psv_wire.psv_idx = psv_index[1]; 1733 1734 mr->sig->sig_status_checked = true; 1735 mr->sig->sig_err_exists = false; 1736 /* Next UMR, Arm SIGERR */ 1737 ++mr->sig->sigerr_count; 1738 } else { 1739 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); 1740 err = -EINVAL; 1741 goto err_free_in; 1742 } 1743 1744 MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3); 1745 MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7); 1746 MLX5_SET(mkc, mkc, umr_en, 1); 1747 1748 mr->ibmr.device = pd->device; 1749 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); 1750 if (err) 1751 goto err_destroy_psv; 1752 1753 mr->mmkey.type = MLX5_MKEY_MR; 1754 mr->ibmr.lkey = mr->mmkey.key; 1755 mr->ibmr.rkey = mr->mmkey.key; 1756 mr->umem = NULL; 1757 kfree(in); 1758 1759 return &mr->ibmr; 1760 1761 err_destroy_psv: 1762 if (mr->sig) { 1763 if (mlx5_core_destroy_psv(dev->mdev, 1764 mr->sig->psv_memory.psv_idx)) 1765 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1766 mr->sig->psv_memory.psv_idx); 1767 if (mlx5_core_destroy_psv(dev->mdev, 1768 mr->sig->psv_wire.psv_idx)) 1769 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1770 mr->sig->psv_wire.psv_idx); 1771 } 1772 mlx5_free_priv_descs(mr); 1773 err_free_sig: 1774 kfree(mr->sig); 1775 err_free_in: 1776 kfree(in); 1777 err_free: 1778 kfree(mr); 1779 return ERR_PTR(err); 1780 } 1781 1782 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, 1783 struct ib_udata *udata) 1784 { 1785 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1786 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1787 struct mlx5_ib_mw *mw = NULL; 1788 u32 *in = NULL; 1789 void *mkc; 1790 int ndescs; 1791 int err; 1792 struct mlx5_ib_alloc_mw req = {}; 1793 struct { 1794 __u32 comp_mask; 1795 __u32 response_length; 1796 } resp = {}; 1797 1798 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req))); 1799 if (err) 1800 return ERR_PTR(err); 1801 1802 if (req.comp_mask || req.reserved1 || req.reserved2) 1803 return ERR_PTR(-EOPNOTSUPP); 1804 1805 if (udata->inlen > sizeof(req) && 1806 !ib_is_udata_cleared(udata, sizeof(req), 1807 udata->inlen - sizeof(req))) 1808 return ERR_PTR(-EOPNOTSUPP); 1809 1810 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4); 1811 1812 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 1813 in = kzalloc(inlen, GFP_KERNEL); 1814 if (!mw || !in) { 1815 err = -ENOMEM; 1816 goto free; 1817 } 1818 1819 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1820 1821 MLX5_SET(mkc, mkc, free, 1); 1822 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 1823 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1824 MLX5_SET(mkc, mkc, umr_en, 1); 1825 MLX5_SET(mkc, mkc, lr, 1); 1826 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 1827 MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2))); 1828 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1829 1830 err = mlx5_core_create_mkey(dev->mdev, &mw->mmkey, in, inlen); 1831 if (err) 1832 goto free; 1833 1834 mw->mmkey.type = MLX5_MKEY_MW; 1835 mw->ibmw.rkey = mw->mmkey.key; 1836 mw->ndescs = ndescs; 1837 1838 resp.response_length = min(offsetof(typeof(resp), response_length) + 1839 sizeof(resp.response_length), udata->outlen); 1840 if (resp.response_length) { 1841 err = ib_copy_to_udata(udata, &resp, resp.response_length); 1842 if (err) { 1843 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey); 1844 goto free; 1845 } 1846 } 1847 1848 kfree(in); 1849 return &mw->ibmw; 1850 1851 free: 1852 kfree(mw); 1853 kfree(in); 1854 return ERR_PTR(err); 1855 } 1856 1857 int mlx5_ib_dealloc_mw(struct ib_mw *mw) 1858 { 1859 struct mlx5_ib_mw *mmw = to_mmw(mw); 1860 int err; 1861 1862 err = mlx5_core_destroy_mkey((to_mdev(mw->device))->mdev, 1863 &mmw->mmkey); 1864 if (!err) 1865 kfree(mmw); 1866 return err; 1867 } 1868 1869 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, 1870 struct ib_mr_status *mr_status) 1871 { 1872 struct mlx5_ib_mr *mmr = to_mmr(ibmr); 1873 int ret = 0; 1874 1875 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) { 1876 pr_err("Invalid status check mask\n"); 1877 ret = -EINVAL; 1878 goto done; 1879 } 1880 1881 mr_status->fail_status = 0; 1882 if (check_mask & IB_MR_CHECK_SIG_STATUS) { 1883 if (!mmr->sig) { 1884 ret = -EINVAL; 1885 pr_err("signature status check requested on a non-signature enabled MR\n"); 1886 goto done; 1887 } 1888 1889 mmr->sig->sig_status_checked = true; 1890 if (!mmr->sig->sig_err_exists) 1891 goto done; 1892 1893 if (ibmr->lkey == mmr->sig->err_item.key) 1894 memcpy(&mr_status->sig_err, &mmr->sig->err_item, 1895 sizeof(mr_status->sig_err)); 1896 else { 1897 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD; 1898 mr_status->sig_err.sig_err_offset = 0; 1899 mr_status->sig_err.key = mmr->sig->err_item.key; 1900 } 1901 1902 mmr->sig->sig_err_exists = false; 1903 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; 1904 } 1905 1906 done: 1907 return ret; 1908 } 1909 1910 static int 1911 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, 1912 struct scatterlist *sgl, 1913 unsigned short sg_nents, 1914 unsigned int *sg_offset_p) 1915 { 1916 struct scatterlist *sg = sgl; 1917 struct mlx5_klm *klms = mr->descs; 1918 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1919 u32 lkey = mr->ibmr.pd->local_dma_lkey; 1920 int i; 1921 1922 mr->ibmr.iova = sg_dma_address(sg) + sg_offset; 1923 mr->ibmr.length = 0; 1924 1925 for_each_sg(sgl, sg, sg_nents, i) { 1926 if (unlikely(i >= mr->max_descs)) 1927 break; 1928 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset); 1929 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset); 1930 klms[i].key = cpu_to_be32(lkey); 1931 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 1932 1933 sg_offset = 0; 1934 } 1935 mr->ndescs = i; 1936 1937 if (sg_offset_p) 1938 *sg_offset_p = sg_offset; 1939 1940 return i; 1941 } 1942 1943 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) 1944 { 1945 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1946 __be64 *descs; 1947 1948 if (unlikely(mr->ndescs == mr->max_descs)) 1949 return -ENOMEM; 1950 1951 descs = mr->descs; 1952 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 1953 1954 return 0; 1955 } 1956 1957 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1958 unsigned int *sg_offset) 1959 { 1960 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1961 int n; 1962 1963 mr->ndescs = 0; 1964 1965 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, 1966 mr->desc_size * mr->max_descs, 1967 DMA_TO_DEVICE); 1968 1969 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 1970 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset); 1971 else 1972 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, 1973 mlx5_set_page); 1974 1975 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map, 1976 mr->desc_size * mr->max_descs, 1977 DMA_TO_DEVICE); 1978 1979 return n; 1980 } 1981