1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 34 #include <linux/kref.h> 35 #include <linux/random.h> 36 #include <linux/debugfs.h> 37 #include <linux/export.h> 38 #include <linux/delay.h> 39 #include <rdma/ib_umem.h> 40 #include <rdma/ib_umem_odp.h> 41 #include <rdma/ib_verbs.h> 42 #include "mlx5_ib.h" 43 44 enum { 45 MAX_PENDING_REG_MR = 8, 46 }; 47 48 #define MLX5_UMR_ALIGN 2048 49 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 50 static __be64 mlx5_ib_update_mtt_emergency_buffer[ 51 MLX5_UMR_MTT_MIN_CHUNK_SIZE/sizeof(__be64)] 52 __aligned(MLX5_UMR_ALIGN); 53 static DEFINE_MUTEX(mlx5_ib_update_mtt_emergency_buffer_mutex); 54 #endif 55 56 static int clean_mr(struct mlx5_ib_mr *mr); 57 58 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 59 { 60 int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr); 61 62 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 63 /* Wait until all page fault handlers using the mr complete. */ 64 synchronize_srcu(&dev->mr_srcu); 65 #endif 66 67 return err; 68 } 69 70 static int order2idx(struct mlx5_ib_dev *dev, int order) 71 { 72 struct mlx5_mr_cache *cache = &dev->cache; 73 74 if (order < cache->ent[0].order) 75 return 0; 76 else 77 return order - cache->ent[0].order; 78 } 79 80 static void reg_mr_callback(int status, void *context) 81 { 82 struct mlx5_ib_mr *mr = context; 83 struct mlx5_ib_dev *dev = mr->dev; 84 struct mlx5_mr_cache *cache = &dev->cache; 85 int c = order2idx(dev, mr->order); 86 struct mlx5_cache_ent *ent = &cache->ent[c]; 87 u8 key; 88 unsigned long flags; 89 struct mlx5_mr_table *table = &dev->mdev->priv.mr_table; 90 int err; 91 92 spin_lock_irqsave(&ent->lock, flags); 93 ent->pending--; 94 spin_unlock_irqrestore(&ent->lock, flags); 95 if (status) { 96 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status); 97 kfree(mr); 98 dev->fill_delay = 1; 99 mod_timer(&dev->delay_timer, jiffies + HZ); 100 return; 101 } 102 103 if (mr->out.hdr.status) { 104 mlx5_ib_warn(dev, "failed - status %d, syndorme 0x%x\n", 105 mr->out.hdr.status, 106 be32_to_cpu(mr->out.hdr.syndrome)); 107 kfree(mr); 108 dev->fill_delay = 1; 109 mod_timer(&dev->delay_timer, jiffies + HZ); 110 return; 111 } 112 113 spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags); 114 key = dev->mdev->priv.mkey_key++; 115 spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags); 116 mr->mmr.key = mlx5_idx_to_mkey(be32_to_cpu(mr->out.mkey) & 0xffffff) | key; 117 118 cache->last_add = jiffies; 119 120 spin_lock_irqsave(&ent->lock, flags); 121 list_add_tail(&mr->list, &ent->head); 122 ent->cur++; 123 ent->size++; 124 spin_unlock_irqrestore(&ent->lock, flags); 125 126 write_lock_irqsave(&table->lock, flags); 127 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmr.key), 128 &mr->mmr); 129 if (err) 130 pr_err("Error inserting to mr tree. 0x%x\n", -err); 131 write_unlock_irqrestore(&table->lock, flags); 132 } 133 134 static int add_keys(struct mlx5_ib_dev *dev, int c, int num) 135 { 136 struct mlx5_mr_cache *cache = &dev->cache; 137 struct mlx5_cache_ent *ent = &cache->ent[c]; 138 struct mlx5_create_mkey_mbox_in *in; 139 struct mlx5_ib_mr *mr; 140 int npages = 1 << ent->order; 141 int err = 0; 142 int i; 143 144 in = kzalloc(sizeof(*in), GFP_KERNEL); 145 if (!in) 146 return -ENOMEM; 147 148 for (i = 0; i < num; i++) { 149 if (ent->pending >= MAX_PENDING_REG_MR) { 150 err = -EAGAIN; 151 break; 152 } 153 154 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 155 if (!mr) { 156 err = -ENOMEM; 157 break; 158 } 159 mr->order = ent->order; 160 mr->umred = 1; 161 mr->dev = dev; 162 in->seg.status = MLX5_MKEY_STATUS_FREE; 163 in->seg.xlt_oct_size = cpu_to_be32((npages + 1) / 2); 164 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 165 in->seg.flags = MLX5_ACCESS_MODE_MTT | MLX5_PERM_UMR_EN; 166 in->seg.log2_page_size = 12; 167 168 spin_lock_irq(&ent->lock); 169 ent->pending++; 170 spin_unlock_irq(&ent->lock); 171 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, 172 sizeof(*in), reg_mr_callback, 173 mr, &mr->out); 174 if (err) { 175 spin_lock_irq(&ent->lock); 176 ent->pending--; 177 spin_unlock_irq(&ent->lock); 178 mlx5_ib_warn(dev, "create mkey failed %d\n", err); 179 kfree(mr); 180 break; 181 } 182 } 183 184 kfree(in); 185 return err; 186 } 187 188 static void remove_keys(struct mlx5_ib_dev *dev, int c, int num) 189 { 190 struct mlx5_mr_cache *cache = &dev->cache; 191 struct mlx5_cache_ent *ent = &cache->ent[c]; 192 struct mlx5_ib_mr *mr; 193 int err; 194 int i; 195 196 for (i = 0; i < num; i++) { 197 spin_lock_irq(&ent->lock); 198 if (list_empty(&ent->head)) { 199 spin_unlock_irq(&ent->lock); 200 return; 201 } 202 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 203 list_del(&mr->list); 204 ent->cur--; 205 ent->size--; 206 spin_unlock_irq(&ent->lock); 207 err = destroy_mkey(dev, mr); 208 if (err) 209 mlx5_ib_warn(dev, "failed destroy mkey\n"); 210 else 211 kfree(mr); 212 } 213 } 214 215 static ssize_t size_write(struct file *filp, const char __user *buf, 216 size_t count, loff_t *pos) 217 { 218 struct mlx5_cache_ent *ent = filp->private_data; 219 struct mlx5_ib_dev *dev = ent->dev; 220 char lbuf[20]; 221 u32 var; 222 int err; 223 int c; 224 225 if (copy_from_user(lbuf, buf, sizeof(lbuf))) 226 return -EFAULT; 227 228 c = order2idx(dev, ent->order); 229 lbuf[sizeof(lbuf) - 1] = 0; 230 231 if (sscanf(lbuf, "%u", &var) != 1) 232 return -EINVAL; 233 234 if (var < ent->limit) 235 return -EINVAL; 236 237 if (var > ent->size) { 238 do { 239 err = add_keys(dev, c, var - ent->size); 240 if (err && err != -EAGAIN) 241 return err; 242 243 usleep_range(3000, 5000); 244 } while (err); 245 } else if (var < ent->size) { 246 remove_keys(dev, c, ent->size - var); 247 } 248 249 return count; 250 } 251 252 static ssize_t size_read(struct file *filp, char __user *buf, size_t count, 253 loff_t *pos) 254 { 255 struct mlx5_cache_ent *ent = filp->private_data; 256 char lbuf[20]; 257 int err; 258 259 if (*pos) 260 return 0; 261 262 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size); 263 if (err < 0) 264 return err; 265 266 if (copy_to_user(buf, lbuf, err)) 267 return -EFAULT; 268 269 *pos += err; 270 271 return err; 272 } 273 274 static const struct file_operations size_fops = { 275 .owner = THIS_MODULE, 276 .open = simple_open, 277 .write = size_write, 278 .read = size_read, 279 }; 280 281 static ssize_t limit_write(struct file *filp, const char __user *buf, 282 size_t count, loff_t *pos) 283 { 284 struct mlx5_cache_ent *ent = filp->private_data; 285 struct mlx5_ib_dev *dev = ent->dev; 286 char lbuf[20]; 287 u32 var; 288 int err; 289 int c; 290 291 if (copy_from_user(lbuf, buf, sizeof(lbuf))) 292 return -EFAULT; 293 294 c = order2idx(dev, ent->order); 295 lbuf[sizeof(lbuf) - 1] = 0; 296 297 if (sscanf(lbuf, "%u", &var) != 1) 298 return -EINVAL; 299 300 if (var > ent->size) 301 return -EINVAL; 302 303 ent->limit = var; 304 305 if (ent->cur < ent->limit) { 306 err = add_keys(dev, c, 2 * ent->limit - ent->cur); 307 if (err) 308 return err; 309 } 310 311 return count; 312 } 313 314 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count, 315 loff_t *pos) 316 { 317 struct mlx5_cache_ent *ent = filp->private_data; 318 char lbuf[20]; 319 int err; 320 321 if (*pos) 322 return 0; 323 324 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); 325 if (err < 0) 326 return err; 327 328 if (copy_to_user(buf, lbuf, err)) 329 return -EFAULT; 330 331 *pos += err; 332 333 return err; 334 } 335 336 static const struct file_operations limit_fops = { 337 .owner = THIS_MODULE, 338 .open = simple_open, 339 .write = limit_write, 340 .read = limit_read, 341 }; 342 343 static int someone_adding(struct mlx5_mr_cache *cache) 344 { 345 int i; 346 347 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 348 if (cache->ent[i].cur < cache->ent[i].limit) 349 return 1; 350 } 351 352 return 0; 353 } 354 355 static void __cache_work_func(struct mlx5_cache_ent *ent) 356 { 357 struct mlx5_ib_dev *dev = ent->dev; 358 struct mlx5_mr_cache *cache = &dev->cache; 359 int i = order2idx(dev, ent->order); 360 int err; 361 362 if (cache->stopped) 363 return; 364 365 ent = &dev->cache.ent[i]; 366 if (ent->cur < 2 * ent->limit && !dev->fill_delay) { 367 err = add_keys(dev, i, 1); 368 if (ent->cur < 2 * ent->limit) { 369 if (err == -EAGAIN) { 370 mlx5_ib_dbg(dev, "returned eagain, order %d\n", 371 i + 2); 372 queue_delayed_work(cache->wq, &ent->dwork, 373 msecs_to_jiffies(3)); 374 } else if (err) { 375 mlx5_ib_warn(dev, "command failed order %d, err %d\n", 376 i + 2, err); 377 queue_delayed_work(cache->wq, &ent->dwork, 378 msecs_to_jiffies(1000)); 379 } else { 380 queue_work(cache->wq, &ent->work); 381 } 382 } 383 } else if (ent->cur > 2 * ent->limit) { 384 if (!someone_adding(cache) && 385 time_after(jiffies, cache->last_add + 300 * HZ)) { 386 remove_keys(dev, i, 1); 387 if (ent->cur > ent->limit) 388 queue_work(cache->wq, &ent->work); 389 } else { 390 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ); 391 } 392 } 393 } 394 395 static void delayed_cache_work_func(struct work_struct *work) 396 { 397 struct mlx5_cache_ent *ent; 398 399 ent = container_of(work, struct mlx5_cache_ent, dwork.work); 400 __cache_work_func(ent); 401 } 402 403 static void cache_work_func(struct work_struct *work) 404 { 405 struct mlx5_cache_ent *ent; 406 407 ent = container_of(work, struct mlx5_cache_ent, work); 408 __cache_work_func(ent); 409 } 410 411 static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order) 412 { 413 struct mlx5_mr_cache *cache = &dev->cache; 414 struct mlx5_ib_mr *mr = NULL; 415 struct mlx5_cache_ent *ent; 416 int c; 417 int i; 418 419 c = order2idx(dev, order); 420 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) { 421 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c); 422 return NULL; 423 } 424 425 for (i = c; i < MAX_MR_CACHE_ENTRIES; i++) { 426 ent = &cache->ent[i]; 427 428 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i); 429 430 spin_lock_irq(&ent->lock); 431 if (!list_empty(&ent->head)) { 432 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, 433 list); 434 list_del(&mr->list); 435 ent->cur--; 436 spin_unlock_irq(&ent->lock); 437 if (ent->cur < ent->limit) 438 queue_work(cache->wq, &ent->work); 439 break; 440 } 441 spin_unlock_irq(&ent->lock); 442 443 queue_work(cache->wq, &ent->work); 444 } 445 446 if (!mr) 447 cache->ent[c].miss++; 448 449 return mr; 450 } 451 452 static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 453 { 454 struct mlx5_mr_cache *cache = &dev->cache; 455 struct mlx5_cache_ent *ent; 456 int shrink = 0; 457 int c; 458 459 c = order2idx(dev, mr->order); 460 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) { 461 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c); 462 return; 463 } 464 ent = &cache->ent[c]; 465 spin_lock_irq(&ent->lock); 466 list_add_tail(&mr->list, &ent->head); 467 ent->cur++; 468 if (ent->cur > 2 * ent->limit) 469 shrink = 1; 470 spin_unlock_irq(&ent->lock); 471 472 if (shrink) 473 queue_work(cache->wq, &ent->work); 474 } 475 476 static void clean_keys(struct mlx5_ib_dev *dev, int c) 477 { 478 struct mlx5_mr_cache *cache = &dev->cache; 479 struct mlx5_cache_ent *ent = &cache->ent[c]; 480 struct mlx5_ib_mr *mr; 481 int err; 482 483 cancel_delayed_work(&ent->dwork); 484 while (1) { 485 spin_lock_irq(&ent->lock); 486 if (list_empty(&ent->head)) { 487 spin_unlock_irq(&ent->lock); 488 return; 489 } 490 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 491 list_del(&mr->list); 492 ent->cur--; 493 ent->size--; 494 spin_unlock_irq(&ent->lock); 495 err = destroy_mkey(dev, mr); 496 if (err) 497 mlx5_ib_warn(dev, "failed destroy mkey\n"); 498 else 499 kfree(mr); 500 } 501 } 502 503 static int mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev) 504 { 505 struct mlx5_mr_cache *cache = &dev->cache; 506 struct mlx5_cache_ent *ent; 507 int i; 508 509 if (!mlx5_debugfs_root) 510 return 0; 511 512 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root); 513 if (!cache->root) 514 return -ENOMEM; 515 516 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 517 ent = &cache->ent[i]; 518 sprintf(ent->name, "%d", ent->order); 519 ent->dir = debugfs_create_dir(ent->name, cache->root); 520 if (!ent->dir) 521 return -ENOMEM; 522 523 ent->fsize = debugfs_create_file("size", 0600, ent->dir, ent, 524 &size_fops); 525 if (!ent->fsize) 526 return -ENOMEM; 527 528 ent->flimit = debugfs_create_file("limit", 0600, ent->dir, ent, 529 &limit_fops); 530 if (!ent->flimit) 531 return -ENOMEM; 532 533 ent->fcur = debugfs_create_u32("cur", 0400, ent->dir, 534 &ent->cur); 535 if (!ent->fcur) 536 return -ENOMEM; 537 538 ent->fmiss = debugfs_create_u32("miss", 0600, ent->dir, 539 &ent->miss); 540 if (!ent->fmiss) 541 return -ENOMEM; 542 } 543 544 return 0; 545 } 546 547 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) 548 { 549 if (!mlx5_debugfs_root) 550 return; 551 552 debugfs_remove_recursive(dev->cache.root); 553 } 554 555 static void delay_time_func(unsigned long ctx) 556 { 557 struct mlx5_ib_dev *dev = (struct mlx5_ib_dev *)ctx; 558 559 dev->fill_delay = 0; 560 } 561 562 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) 563 { 564 struct mlx5_mr_cache *cache = &dev->cache; 565 struct mlx5_cache_ent *ent; 566 int limit; 567 int err; 568 int i; 569 570 cache->wq = create_singlethread_workqueue("mkey_cache"); 571 if (!cache->wq) { 572 mlx5_ib_warn(dev, "failed to create work queue\n"); 573 return -ENOMEM; 574 } 575 576 setup_timer(&dev->delay_timer, delay_time_func, (unsigned long)dev); 577 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 578 INIT_LIST_HEAD(&cache->ent[i].head); 579 spin_lock_init(&cache->ent[i].lock); 580 581 ent = &cache->ent[i]; 582 INIT_LIST_HEAD(&ent->head); 583 spin_lock_init(&ent->lock); 584 ent->order = i + 2; 585 ent->dev = dev; 586 587 if (dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) 588 limit = dev->mdev->profile->mr_cache[i].limit; 589 else 590 limit = 0; 591 592 INIT_WORK(&ent->work, cache_work_func); 593 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 594 ent->limit = limit; 595 queue_work(cache->wq, &ent->work); 596 } 597 598 err = mlx5_mr_cache_debugfs_init(dev); 599 if (err) 600 mlx5_ib_warn(dev, "cache debugfs failure\n"); 601 602 return 0; 603 } 604 605 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) 606 { 607 int i; 608 609 dev->cache.stopped = 1; 610 flush_workqueue(dev->cache.wq); 611 612 mlx5_mr_cache_debugfs_cleanup(dev); 613 614 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) 615 clean_keys(dev, i); 616 617 destroy_workqueue(dev->cache.wq); 618 del_timer_sync(&dev->delay_timer); 619 620 return 0; 621 } 622 623 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) 624 { 625 struct mlx5_ib_dev *dev = to_mdev(pd->device); 626 struct mlx5_core_dev *mdev = dev->mdev; 627 struct mlx5_create_mkey_mbox_in *in; 628 struct mlx5_mkey_seg *seg; 629 struct mlx5_ib_mr *mr; 630 int err; 631 632 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 633 if (!mr) 634 return ERR_PTR(-ENOMEM); 635 636 in = kzalloc(sizeof(*in), GFP_KERNEL); 637 if (!in) { 638 err = -ENOMEM; 639 goto err_free; 640 } 641 642 seg = &in->seg; 643 seg->flags = convert_access(acc) | MLX5_ACCESS_MODE_PA; 644 seg->flags_pd = cpu_to_be32(to_mpd(pd)->pdn | MLX5_MKEY_LEN64); 645 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 646 seg->start_addr = 0; 647 648 err = mlx5_core_create_mkey(mdev, &mr->mmr, in, sizeof(*in), NULL, NULL, 649 NULL); 650 if (err) 651 goto err_in; 652 653 kfree(in); 654 mr->ibmr.lkey = mr->mmr.key; 655 mr->ibmr.rkey = mr->mmr.key; 656 mr->umem = NULL; 657 658 return &mr->ibmr; 659 660 err_in: 661 kfree(in); 662 663 err_free: 664 kfree(mr); 665 666 return ERR_PTR(err); 667 } 668 669 static int get_octo_len(u64 addr, u64 len, int page_size) 670 { 671 u64 offset; 672 int npages; 673 674 offset = addr & (page_size - 1); 675 npages = ALIGN(len + offset, page_size) >> ilog2(page_size); 676 return (npages + 1) / 2; 677 } 678 679 static int use_umr(int order) 680 { 681 return order <= MLX5_MAX_UMR_SHIFT; 682 } 683 684 static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr, 685 struct ib_sge *sg, u64 dma, int n, u32 key, 686 int page_shift, u64 virt_addr, u64 len, 687 int access_flags) 688 { 689 struct mlx5_ib_dev *dev = to_mdev(pd->device); 690 struct mlx5_umr_wr *umrwr = umr_wr(wr); 691 692 sg->addr = dma; 693 sg->length = ALIGN(sizeof(u64) * n, 64); 694 sg->lkey = dev->umrc.pd->local_dma_lkey; 695 696 wr->next = NULL; 697 wr->send_flags = 0; 698 wr->sg_list = sg; 699 if (n) 700 wr->num_sge = 1; 701 else 702 wr->num_sge = 0; 703 704 wr->opcode = MLX5_IB_WR_UMR; 705 706 umrwr->npages = n; 707 umrwr->page_shift = page_shift; 708 umrwr->mkey = key; 709 umrwr->target.virt_addr = virt_addr; 710 umrwr->length = len; 711 umrwr->access_flags = access_flags; 712 umrwr->pd = pd; 713 } 714 715 static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev, 716 struct ib_send_wr *wr, u32 key) 717 { 718 struct mlx5_umr_wr *umrwr = umr_wr(wr); 719 720 wr->send_flags = MLX5_IB_SEND_UMR_UNREG | MLX5_IB_SEND_UMR_FAIL_IF_FREE; 721 wr->opcode = MLX5_IB_WR_UMR; 722 umrwr->mkey = key; 723 } 724 725 void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context) 726 { 727 struct mlx5_ib_umr_context *context; 728 struct ib_wc wc; 729 int err; 730 731 while (1) { 732 err = ib_poll_cq(cq, 1, &wc); 733 if (err < 0) { 734 pr_warn("poll cq error %d\n", err); 735 return; 736 } 737 if (err == 0) 738 break; 739 740 context = (struct mlx5_ib_umr_context *) (unsigned long) wc.wr_id; 741 context->status = wc.status; 742 complete(&context->done); 743 } 744 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 745 } 746 747 static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem, 748 u64 virt_addr, u64 len, int npages, 749 int page_shift, int order, int access_flags) 750 { 751 struct mlx5_ib_dev *dev = to_mdev(pd->device); 752 struct device *ddev = dev->ib_dev.dma_device; 753 struct umr_common *umrc = &dev->umrc; 754 struct mlx5_ib_umr_context umr_context; 755 struct mlx5_umr_wr umrwr; 756 struct ib_send_wr *bad; 757 struct mlx5_ib_mr *mr; 758 struct ib_sge sg; 759 int size; 760 __be64 *mr_pas; 761 __be64 *pas; 762 dma_addr_t dma; 763 int err = 0; 764 int i; 765 766 for (i = 0; i < 1; i++) { 767 mr = alloc_cached_mr(dev, order); 768 if (mr) 769 break; 770 771 err = add_keys(dev, order2idx(dev, order), 1); 772 if (err && err != -EAGAIN) { 773 mlx5_ib_warn(dev, "add_keys failed, err %d\n", err); 774 break; 775 } 776 } 777 778 if (!mr) 779 return ERR_PTR(-EAGAIN); 780 781 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes. 782 * To avoid copying garbage after the pas array, we allocate 783 * a little more. */ 784 size = ALIGN(sizeof(u64) * npages, MLX5_UMR_MTT_ALIGNMENT); 785 mr_pas = kmalloc(size + MLX5_UMR_ALIGN - 1, GFP_KERNEL); 786 if (!mr_pas) { 787 err = -ENOMEM; 788 goto free_mr; 789 } 790 791 pas = PTR_ALIGN(mr_pas, MLX5_UMR_ALIGN); 792 mlx5_ib_populate_pas(dev, umem, page_shift, pas, MLX5_IB_MTT_PRESENT); 793 /* Clear padding after the actual pages. */ 794 memset(pas + npages, 0, size - npages * sizeof(u64)); 795 796 dma = dma_map_single(ddev, pas, size, DMA_TO_DEVICE); 797 if (dma_mapping_error(ddev, dma)) { 798 err = -ENOMEM; 799 goto free_pas; 800 } 801 802 memset(&umrwr, 0, sizeof(umrwr)); 803 umrwr.wr.wr_id = (u64)(unsigned long)&umr_context; 804 prep_umr_reg_wqe(pd, &umrwr.wr, &sg, dma, npages, mr->mmr.key, 805 page_shift, virt_addr, len, access_flags); 806 807 mlx5_ib_init_umr_context(&umr_context); 808 down(&umrc->sem); 809 err = ib_post_send(umrc->qp, &umrwr.wr, &bad); 810 if (err) { 811 mlx5_ib_warn(dev, "post send failed, err %d\n", err); 812 goto unmap_dma; 813 } else { 814 wait_for_completion(&umr_context.done); 815 if (umr_context.status != IB_WC_SUCCESS) { 816 mlx5_ib_warn(dev, "reg umr failed\n"); 817 err = -EFAULT; 818 } 819 } 820 821 mr->mmr.iova = virt_addr; 822 mr->mmr.size = len; 823 mr->mmr.pd = to_mpd(pd)->pdn; 824 825 mr->live = 1; 826 827 unmap_dma: 828 up(&umrc->sem); 829 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE); 830 831 free_pas: 832 kfree(mr_pas); 833 834 free_mr: 835 if (err) { 836 free_cached_mr(dev, mr); 837 return ERR_PTR(err); 838 } 839 840 return mr; 841 } 842 843 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 844 int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index, int npages, 845 int zap) 846 { 847 struct mlx5_ib_dev *dev = mr->dev; 848 struct device *ddev = dev->ib_dev.dma_device; 849 struct umr_common *umrc = &dev->umrc; 850 struct mlx5_ib_umr_context umr_context; 851 struct ib_umem *umem = mr->umem; 852 int size; 853 __be64 *pas; 854 dma_addr_t dma; 855 struct ib_send_wr *bad; 856 struct mlx5_umr_wr wr; 857 struct ib_sge sg; 858 int err = 0; 859 const int page_index_alignment = MLX5_UMR_MTT_ALIGNMENT / sizeof(u64); 860 const int page_index_mask = page_index_alignment - 1; 861 size_t pages_mapped = 0; 862 size_t pages_to_map = 0; 863 size_t pages_iter = 0; 864 int use_emergency_buf = 0; 865 866 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes, 867 * so we need to align the offset and length accordingly */ 868 if (start_page_index & page_index_mask) { 869 npages += start_page_index & page_index_mask; 870 start_page_index &= ~page_index_mask; 871 } 872 873 pages_to_map = ALIGN(npages, page_index_alignment); 874 875 if (start_page_index + pages_to_map > MLX5_MAX_UMR_PAGES) 876 return -EINVAL; 877 878 size = sizeof(u64) * pages_to_map; 879 size = min_t(int, PAGE_SIZE, size); 880 /* We allocate with GFP_ATOMIC to avoid recursion into page-reclaim 881 * code, when we are called from an invalidation. The pas buffer must 882 * be 2k-aligned for Connect-IB. */ 883 pas = (__be64 *)get_zeroed_page(GFP_ATOMIC); 884 if (!pas) { 885 mlx5_ib_warn(dev, "unable to allocate memory during MTT update, falling back to slower chunked mechanism.\n"); 886 pas = mlx5_ib_update_mtt_emergency_buffer; 887 size = MLX5_UMR_MTT_MIN_CHUNK_SIZE; 888 use_emergency_buf = 1; 889 mutex_lock(&mlx5_ib_update_mtt_emergency_buffer_mutex); 890 memset(pas, 0, size); 891 } 892 pages_iter = size / sizeof(u64); 893 dma = dma_map_single(ddev, pas, size, DMA_TO_DEVICE); 894 if (dma_mapping_error(ddev, dma)) { 895 mlx5_ib_err(dev, "unable to map DMA during MTT update.\n"); 896 err = -ENOMEM; 897 goto free_pas; 898 } 899 900 for (pages_mapped = 0; 901 pages_mapped < pages_to_map && !err; 902 pages_mapped += pages_iter, start_page_index += pages_iter) { 903 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE); 904 905 npages = min_t(size_t, 906 pages_iter, 907 ib_umem_num_pages(umem) - start_page_index); 908 909 if (!zap) { 910 __mlx5_ib_populate_pas(dev, umem, PAGE_SHIFT, 911 start_page_index, npages, pas, 912 MLX5_IB_MTT_PRESENT); 913 /* Clear padding after the pages brought from the 914 * umem. */ 915 memset(pas + npages, 0, size - npages * sizeof(u64)); 916 } 917 918 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE); 919 920 memset(&wr, 0, sizeof(wr)); 921 wr.wr.wr_id = (u64)(unsigned long)&umr_context; 922 923 sg.addr = dma; 924 sg.length = ALIGN(npages * sizeof(u64), 925 MLX5_UMR_MTT_ALIGNMENT); 926 sg.lkey = dev->umrc.pd->local_dma_lkey; 927 928 wr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE | 929 MLX5_IB_SEND_UMR_UPDATE_MTT; 930 wr.wr.sg_list = &sg; 931 wr.wr.num_sge = 1; 932 wr.wr.opcode = MLX5_IB_WR_UMR; 933 wr.npages = sg.length / sizeof(u64); 934 wr.page_shift = PAGE_SHIFT; 935 wr.mkey = mr->mmr.key; 936 wr.target.offset = start_page_index; 937 938 mlx5_ib_init_umr_context(&umr_context); 939 down(&umrc->sem); 940 err = ib_post_send(umrc->qp, &wr.wr, &bad); 941 if (err) { 942 mlx5_ib_err(dev, "UMR post send failed, err %d\n", err); 943 } else { 944 wait_for_completion(&umr_context.done); 945 if (umr_context.status != IB_WC_SUCCESS) { 946 mlx5_ib_err(dev, "UMR completion failed, code %d\n", 947 umr_context.status); 948 err = -EFAULT; 949 } 950 } 951 up(&umrc->sem); 952 } 953 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE); 954 955 free_pas: 956 if (!use_emergency_buf) 957 free_page((unsigned long)pas); 958 else 959 mutex_unlock(&mlx5_ib_update_mtt_emergency_buffer_mutex); 960 961 return err; 962 } 963 #endif 964 965 static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr, 966 u64 length, struct ib_umem *umem, 967 int npages, int page_shift, 968 int access_flags) 969 { 970 struct mlx5_ib_dev *dev = to_mdev(pd->device); 971 struct mlx5_create_mkey_mbox_in *in; 972 struct mlx5_ib_mr *mr; 973 int inlen; 974 int err; 975 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 976 977 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 978 if (!mr) 979 return ERR_PTR(-ENOMEM); 980 981 inlen = sizeof(*in) + sizeof(*in->pas) * ((npages + 1) / 2) * 2; 982 in = mlx5_vzalloc(inlen); 983 if (!in) { 984 err = -ENOMEM; 985 goto err_1; 986 } 987 mlx5_ib_populate_pas(dev, umem, page_shift, in->pas, 988 pg_cap ? MLX5_IB_MTT_PRESENT : 0); 989 990 /* The MLX5_MKEY_INBOX_PG_ACCESS bit allows setting the access flags 991 * in the page list submitted with the command. */ 992 in->flags = pg_cap ? cpu_to_be32(MLX5_MKEY_INBOX_PG_ACCESS) : 0; 993 in->seg.flags = convert_access(access_flags) | 994 MLX5_ACCESS_MODE_MTT; 995 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn); 996 in->seg.start_addr = cpu_to_be64(virt_addr); 997 in->seg.len = cpu_to_be64(length); 998 in->seg.bsfs_octo_size = 0; 999 in->seg.xlt_oct_size = cpu_to_be32(get_octo_len(virt_addr, length, 1 << page_shift)); 1000 in->seg.log2_page_size = page_shift; 1001 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 1002 in->xlat_oct_act_size = cpu_to_be32(get_octo_len(virt_addr, length, 1003 1 << page_shift)); 1004 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, inlen, NULL, 1005 NULL, NULL); 1006 if (err) { 1007 mlx5_ib_warn(dev, "create mkey failed\n"); 1008 goto err_2; 1009 } 1010 mr->umem = umem; 1011 mr->dev = dev; 1012 mr->live = 1; 1013 kvfree(in); 1014 1015 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmr.key); 1016 1017 return mr; 1018 1019 err_2: 1020 kvfree(in); 1021 1022 err_1: 1023 kfree(mr); 1024 1025 return ERR_PTR(err); 1026 } 1027 1028 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 1029 u64 virt_addr, int access_flags, 1030 struct ib_udata *udata) 1031 { 1032 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1033 struct mlx5_ib_mr *mr = NULL; 1034 struct ib_umem *umem; 1035 int page_shift; 1036 int npages; 1037 int ncont; 1038 int order; 1039 int err; 1040 1041 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1042 start, virt_addr, length, access_flags); 1043 umem = ib_umem_get(pd->uobject->context, start, length, access_flags, 1044 0); 1045 if (IS_ERR(umem)) { 1046 mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(umem)); 1047 return (void *)umem; 1048 } 1049 1050 mlx5_ib_cont_pages(umem, start, &npages, &page_shift, &ncont, &order); 1051 if (!npages) { 1052 mlx5_ib_warn(dev, "avoid zero region\n"); 1053 err = -EINVAL; 1054 goto error; 1055 } 1056 1057 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n", 1058 npages, ncont, order, page_shift); 1059 1060 if (use_umr(order)) { 1061 mr = reg_umr(pd, umem, virt_addr, length, ncont, page_shift, 1062 order, access_flags); 1063 if (PTR_ERR(mr) == -EAGAIN) { 1064 mlx5_ib_dbg(dev, "cache empty for order %d", order); 1065 mr = NULL; 1066 } 1067 } else if (access_flags & IB_ACCESS_ON_DEMAND) { 1068 err = -EINVAL; 1069 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB"); 1070 goto error; 1071 } 1072 1073 if (!mr) 1074 mr = reg_create(pd, virt_addr, length, umem, ncont, page_shift, 1075 access_flags); 1076 1077 if (IS_ERR(mr)) { 1078 err = PTR_ERR(mr); 1079 goto error; 1080 } 1081 1082 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmr.key); 1083 1084 mr->umem = umem; 1085 mr->npages = npages; 1086 atomic_add(npages, &dev->mdev->priv.reg_pages); 1087 mr->ibmr.lkey = mr->mmr.key; 1088 mr->ibmr.rkey = mr->mmr.key; 1089 1090 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1091 if (umem->odp_data) { 1092 /* 1093 * This barrier prevents the compiler from moving the 1094 * setting of umem->odp_data->private to point to our 1095 * MR, before reg_umr finished, to ensure that the MR 1096 * initialization have finished before starting to 1097 * handle invalidations. 1098 */ 1099 smp_wmb(); 1100 mr->umem->odp_data->private = mr; 1101 /* 1102 * Make sure we will see the new 1103 * umem->odp_data->private value in the invalidation 1104 * routines, before we can get page faults on the 1105 * MR. Page faults can happen once we put the MR in 1106 * the tree, below this line. Without the barrier, 1107 * there can be a fault handling and an invalidation 1108 * before umem->odp_data->private == mr is visible to 1109 * the invalidation handler. 1110 */ 1111 smp_wmb(); 1112 } 1113 #endif 1114 1115 return &mr->ibmr; 1116 1117 error: 1118 ib_umem_release(umem); 1119 return ERR_PTR(err); 1120 } 1121 1122 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1123 { 1124 struct umr_common *umrc = &dev->umrc; 1125 struct mlx5_ib_umr_context umr_context; 1126 struct mlx5_umr_wr umrwr; 1127 struct ib_send_wr *bad; 1128 int err; 1129 1130 memset(&umrwr.wr, 0, sizeof(umrwr)); 1131 umrwr.wr.wr_id = (u64)(unsigned long)&umr_context; 1132 prep_umr_unreg_wqe(dev, &umrwr.wr, mr->mmr.key); 1133 1134 mlx5_ib_init_umr_context(&umr_context); 1135 down(&umrc->sem); 1136 err = ib_post_send(umrc->qp, &umrwr.wr, &bad); 1137 if (err) { 1138 up(&umrc->sem); 1139 mlx5_ib_dbg(dev, "err %d\n", err); 1140 goto error; 1141 } else { 1142 wait_for_completion(&umr_context.done); 1143 up(&umrc->sem); 1144 } 1145 if (umr_context.status != IB_WC_SUCCESS) { 1146 mlx5_ib_warn(dev, "unreg umr failed\n"); 1147 err = -EFAULT; 1148 goto error; 1149 } 1150 return 0; 1151 1152 error: 1153 return err; 1154 } 1155 1156 static int 1157 mlx5_alloc_priv_descs(struct ib_device *device, 1158 struct mlx5_ib_mr *mr, 1159 int ndescs, 1160 int desc_size) 1161 { 1162 int size = ndescs * desc_size; 1163 int add_size; 1164 int ret; 1165 1166 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0); 1167 1168 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL); 1169 if (!mr->descs_alloc) 1170 return -ENOMEM; 1171 1172 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN); 1173 1174 mr->desc_map = dma_map_single(device->dma_device, mr->descs, 1175 size, DMA_TO_DEVICE); 1176 if (dma_mapping_error(device->dma_device, mr->desc_map)) { 1177 ret = -ENOMEM; 1178 goto err; 1179 } 1180 1181 return 0; 1182 err: 1183 kfree(mr->descs_alloc); 1184 1185 return ret; 1186 } 1187 1188 static void 1189 mlx5_free_priv_descs(struct mlx5_ib_mr *mr) 1190 { 1191 if (mr->descs) { 1192 struct ib_device *device = mr->ibmr.device; 1193 int size = mr->max_descs * mr->desc_size; 1194 1195 dma_unmap_single(device->dma_device, mr->desc_map, 1196 size, DMA_TO_DEVICE); 1197 kfree(mr->descs_alloc); 1198 mr->descs = NULL; 1199 } 1200 } 1201 1202 static int clean_mr(struct mlx5_ib_mr *mr) 1203 { 1204 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); 1205 int umred = mr->umred; 1206 int err; 1207 1208 if (mr->sig) { 1209 if (mlx5_core_destroy_psv(dev->mdev, 1210 mr->sig->psv_memory.psv_idx)) 1211 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1212 mr->sig->psv_memory.psv_idx); 1213 if (mlx5_core_destroy_psv(dev->mdev, 1214 mr->sig->psv_wire.psv_idx)) 1215 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1216 mr->sig->psv_wire.psv_idx); 1217 kfree(mr->sig); 1218 mr->sig = NULL; 1219 } 1220 1221 mlx5_free_priv_descs(mr); 1222 1223 if (!umred) { 1224 err = destroy_mkey(dev, mr); 1225 if (err) { 1226 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n", 1227 mr->mmr.key, err); 1228 return err; 1229 } 1230 } else { 1231 err = unreg_umr(dev, mr); 1232 if (err) { 1233 mlx5_ib_warn(dev, "failed unregister\n"); 1234 return err; 1235 } 1236 free_cached_mr(dev, mr); 1237 } 1238 1239 if (!umred) 1240 kfree(mr); 1241 1242 return 0; 1243 } 1244 1245 int mlx5_ib_dereg_mr(struct ib_mr *ibmr) 1246 { 1247 struct mlx5_ib_dev *dev = to_mdev(ibmr->device); 1248 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1249 int npages = mr->npages; 1250 struct ib_umem *umem = mr->umem; 1251 1252 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1253 if (umem && umem->odp_data) { 1254 /* Prevent new page faults from succeeding */ 1255 mr->live = 0; 1256 /* Wait for all running page-fault handlers to finish. */ 1257 synchronize_srcu(&dev->mr_srcu); 1258 /* Destroy all page mappings */ 1259 mlx5_ib_invalidate_range(umem, ib_umem_start(umem), 1260 ib_umem_end(umem)); 1261 /* 1262 * We kill the umem before the MR for ODP, 1263 * so that there will not be any invalidations in 1264 * flight, looking at the *mr struct. 1265 */ 1266 ib_umem_release(umem); 1267 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1268 1269 /* Avoid double-freeing the umem. */ 1270 umem = NULL; 1271 } 1272 #endif 1273 1274 clean_mr(mr); 1275 1276 if (umem) { 1277 ib_umem_release(umem); 1278 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1279 } 1280 1281 return 0; 1282 } 1283 1284 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, 1285 enum ib_mr_type mr_type, 1286 u32 max_num_sg) 1287 { 1288 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1289 struct mlx5_create_mkey_mbox_in *in; 1290 struct mlx5_ib_mr *mr; 1291 int access_mode, err; 1292 int ndescs = roundup(max_num_sg, 4); 1293 1294 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1295 if (!mr) 1296 return ERR_PTR(-ENOMEM); 1297 1298 in = kzalloc(sizeof(*in), GFP_KERNEL); 1299 if (!in) { 1300 err = -ENOMEM; 1301 goto err_free; 1302 } 1303 1304 in->seg.status = MLX5_MKEY_STATUS_FREE; 1305 in->seg.xlt_oct_size = cpu_to_be32(ndescs); 1306 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); 1307 in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn); 1308 1309 if (mr_type == IB_MR_TYPE_MEM_REG) { 1310 access_mode = MLX5_ACCESS_MODE_MTT; 1311 in->seg.log2_page_size = PAGE_SHIFT; 1312 1313 err = mlx5_alloc_priv_descs(pd->device, mr, 1314 ndescs, sizeof(u64)); 1315 if (err) 1316 goto err_free_in; 1317 1318 mr->desc_size = sizeof(u64); 1319 mr->max_descs = ndescs; 1320 } else if (mr_type == IB_MR_TYPE_SIGNATURE) { 1321 u32 psv_index[2]; 1322 1323 in->seg.flags_pd = cpu_to_be32(be32_to_cpu(in->seg.flags_pd) | 1324 MLX5_MKEY_BSF_EN); 1325 in->seg.bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE); 1326 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); 1327 if (!mr->sig) { 1328 err = -ENOMEM; 1329 goto err_free_in; 1330 } 1331 1332 /* create mem & wire PSVs */ 1333 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 1334 2, psv_index); 1335 if (err) 1336 goto err_free_sig; 1337 1338 access_mode = MLX5_ACCESS_MODE_KLM; 1339 mr->sig->psv_memory.psv_idx = psv_index[0]; 1340 mr->sig->psv_wire.psv_idx = psv_index[1]; 1341 1342 mr->sig->sig_status_checked = true; 1343 mr->sig->sig_err_exists = false; 1344 /* Next UMR, Arm SIGERR */ 1345 ++mr->sig->sigerr_count; 1346 } else { 1347 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); 1348 err = -EINVAL; 1349 goto err_free_in; 1350 } 1351 1352 in->seg.flags = MLX5_PERM_UMR_EN | access_mode; 1353 err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, sizeof(*in), 1354 NULL, NULL, NULL); 1355 if (err) 1356 goto err_destroy_psv; 1357 1358 mr->ibmr.lkey = mr->mmr.key; 1359 mr->ibmr.rkey = mr->mmr.key; 1360 mr->umem = NULL; 1361 kfree(in); 1362 1363 return &mr->ibmr; 1364 1365 err_destroy_psv: 1366 if (mr->sig) { 1367 if (mlx5_core_destroy_psv(dev->mdev, 1368 mr->sig->psv_memory.psv_idx)) 1369 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1370 mr->sig->psv_memory.psv_idx); 1371 if (mlx5_core_destroy_psv(dev->mdev, 1372 mr->sig->psv_wire.psv_idx)) 1373 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1374 mr->sig->psv_wire.psv_idx); 1375 } 1376 mlx5_free_priv_descs(mr); 1377 err_free_sig: 1378 kfree(mr->sig); 1379 err_free_in: 1380 kfree(in); 1381 err_free: 1382 kfree(mr); 1383 return ERR_PTR(err); 1384 } 1385 1386 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, 1387 struct ib_mr_status *mr_status) 1388 { 1389 struct mlx5_ib_mr *mmr = to_mmr(ibmr); 1390 int ret = 0; 1391 1392 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) { 1393 pr_err("Invalid status check mask\n"); 1394 ret = -EINVAL; 1395 goto done; 1396 } 1397 1398 mr_status->fail_status = 0; 1399 if (check_mask & IB_MR_CHECK_SIG_STATUS) { 1400 if (!mmr->sig) { 1401 ret = -EINVAL; 1402 pr_err("signature status check requested on a non-signature enabled MR\n"); 1403 goto done; 1404 } 1405 1406 mmr->sig->sig_status_checked = true; 1407 if (!mmr->sig->sig_err_exists) 1408 goto done; 1409 1410 if (ibmr->lkey == mmr->sig->err_item.key) 1411 memcpy(&mr_status->sig_err, &mmr->sig->err_item, 1412 sizeof(mr_status->sig_err)); 1413 else { 1414 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD; 1415 mr_status->sig_err.sig_err_offset = 0; 1416 mr_status->sig_err.key = mmr->sig->err_item.key; 1417 } 1418 1419 mmr->sig->sig_err_exists = false; 1420 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; 1421 } 1422 1423 done: 1424 return ret; 1425 } 1426 1427 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) 1428 { 1429 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1430 __be64 *descs; 1431 1432 if (unlikely(mr->ndescs == mr->max_descs)) 1433 return -ENOMEM; 1434 1435 descs = mr->descs; 1436 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 1437 1438 return 0; 1439 } 1440 1441 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, 1442 struct scatterlist *sg, 1443 int sg_nents) 1444 { 1445 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1446 int n; 1447 1448 mr->ndescs = 0; 1449 1450 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, 1451 mr->desc_size * mr->max_descs, 1452 DMA_TO_DEVICE); 1453 1454 n = ib_sg_to_pages(ibmr, sg, sg_nents, mlx5_set_page); 1455 1456 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map, 1457 mr->desc_size * mr->max_descs, 1458 DMA_TO_DEVICE); 1459 1460 return n; 1461 } 1462