mr.c (6b7533869523ae58e2b914551305b0e47cbeb247) mr.c (0113780870b1597ae49f30abfa4957c239f913d3)
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2020, Intel Corporation. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the

--- 105 unchanged lines hidden (view full) ---

114 MLX5_SET(create_mkey_in, async_create->in, opcode,
115 MLX5_CMD_OP_CREATE_MKEY);
116 assign_mkey_variant(dev, &async_create->mkey, async_create->in);
117 return mlx5_cmd_exec_cb(&dev->async_ctx, async_create->in, inlen,
118 async_create->out, outlen, create_mkey_callback,
119 &async_create->cb_work);
120}
121
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2020, Intel Corporation. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the

--- 105 unchanged lines hidden (view full) ---

114 MLX5_SET(create_mkey_in, async_create->in, opcode,
115 MLX5_CMD_OP_CREATE_MKEY);
116 assign_mkey_variant(dev, &async_create->mkey, async_create->in);
117 return mlx5_cmd_exec_cb(&dev->async_ctx, async_create->in, inlen,
118 async_create->out, outlen, create_mkey_callback,
119 &async_create->cb_work);
120}
121
122static int mr_cache_max_order(struct mlx5_ib_dev *dev);
122static int mkey_cache_max_order(struct mlx5_ib_dev *dev);
123static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
124
125static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
126{
127 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
128
129 return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key);
130}

--- 379 unchanged lines hidden (view full) ---

510
511static const struct file_operations limit_fops = {
512 .owner = THIS_MODULE,
513 .open = simple_open,
514 .write = limit_write,
515 .read = limit_read,
516};
517
123static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
124
125static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
126{
127 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
128
129 return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key);
130}

--- 379 unchanged lines hidden (view full) ---

510
511static const struct file_operations limit_fops = {
512 .owner = THIS_MODULE,
513 .open = simple_open,
514 .write = limit_write,
515 .read = limit_read,
516};
517
518static bool someone_adding(struct mlx5_mr_cache *cache)
518static bool someone_adding(struct mlx5_mkey_cache *cache)
519{
520 unsigned int i;
521
519{
520 unsigned int i;
521
522 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
522 for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
523 struct mlx5_cache_ent *ent = &cache->ent[i];
524 bool ret;
525
526 xa_lock_irq(&ent->mkeys);
527 ret = ent->stored < ent->limit;
528 xa_unlock_irq(&ent->mkeys);
529 if (ret)
530 return true;

--- 33 unchanged lines hidden (view full) ---

564 else
565 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
566 }
567}
568
569static void __cache_work_func(struct mlx5_cache_ent *ent)
570{
571 struct mlx5_ib_dev *dev = ent->dev;
523 struct mlx5_cache_ent *ent = &cache->ent[i];
524 bool ret;
525
526 xa_lock_irq(&ent->mkeys);
527 ret = ent->stored < ent->limit;
528 xa_unlock_irq(&ent->mkeys);
529 if (ret)
530 return true;

--- 33 unchanged lines hidden (view full) ---

564 else
565 mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);
566 }
567}
568
569static void __cache_work_func(struct mlx5_cache_ent *ent)
570{
571 struct mlx5_ib_dev *dev = ent->dev;
572 struct mlx5_mr_cache *cache = &dev->cache;
572 struct mlx5_mkey_cache *cache = &dev->cache;
573 int err;
574
575 xa_lock_irq(&ent->mkeys);
576 if (ent->disabled)
577 goto out;
578
579 if (ent->fill_to_high_water && ent->reserved < 2 * ent->limit &&
580 !READ_ONCE(dev->fill_delay)) {

--- 95 unchanged lines hidden (view full) ---

676 mr->mmkey.cache_ent = ent;
677 mr->mmkey.type = MLX5_MKEY_MR;
678 init_waitqueue_head(&mr->mmkey.wait);
679 return mr;
680}
681
682static void clean_keys(struct mlx5_ib_dev *dev, int c)
683{
573 int err;
574
575 xa_lock_irq(&ent->mkeys);
576 if (ent->disabled)
577 goto out;
578
579 if (ent->fill_to_high_water && ent->reserved < 2 * ent->limit &&
580 !READ_ONCE(dev->fill_delay)) {

--- 95 unchanged lines hidden (view full) ---

676 mr->mmkey.cache_ent = ent;
677 mr->mmkey.type = MLX5_MKEY_MR;
678 init_waitqueue_head(&mr->mmkey.wait);
679 return mr;
680}
681
682static void clean_keys(struct mlx5_ib_dev *dev, int c)
683{
684 struct mlx5_mr_cache *cache = &dev->cache;
684 struct mlx5_mkey_cache *cache = &dev->cache;
685 struct mlx5_cache_ent *ent = &cache->ent[c];
686 u32 mkey;
687
688 cancel_delayed_work(&ent->dwork);
689 xa_lock_irq(&ent->mkeys);
690 while (ent->stored) {
691 mkey = pop_stored_mkey(ent);
692 xa_unlock_irq(&ent->mkeys);
693 mlx5_core_destroy_mkey(dev->mdev, mkey);
694 xa_lock_irq(&ent->mkeys);
695 }
696 xa_unlock_irq(&ent->mkeys);
697}
698
685 struct mlx5_cache_ent *ent = &cache->ent[c];
686 u32 mkey;
687
688 cancel_delayed_work(&ent->dwork);
689 xa_lock_irq(&ent->mkeys);
690 while (ent->stored) {
691 mkey = pop_stored_mkey(ent);
692 xa_unlock_irq(&ent->mkeys);
693 mlx5_core_destroy_mkey(dev->mdev, mkey);
694 xa_lock_irq(&ent->mkeys);
695 }
696 xa_unlock_irq(&ent->mkeys);
697}
698
699static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
699static void mlx5_mkey_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
700{
701 if (!mlx5_debugfs_root || dev->is_rep)
702 return;
703
704 debugfs_remove_recursive(dev->cache.root);
705 dev->cache.root = NULL;
706}
707
700{
701 if (!mlx5_debugfs_root || dev->is_rep)
702 return;
703
704 debugfs_remove_recursive(dev->cache.root);
705 dev->cache.root = NULL;
706}
707
708static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
708static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev)
709{
709{
710 struct mlx5_mr_cache *cache = &dev->cache;
710 struct mlx5_mkey_cache *cache = &dev->cache;
711 struct mlx5_cache_ent *ent;
712 struct dentry *dir;
713 int i;
714
715 if (!mlx5_debugfs_root || dev->is_rep)
716 return;
717
718 cache->root = debugfs_create_dir("mr_cache", mlx5_debugfs_get_dev_root(dev->mdev));
719
711 struct mlx5_cache_ent *ent;
712 struct dentry *dir;
713 int i;
714
715 if (!mlx5_debugfs_root || dev->is_rep)
716 return;
717
718 cache->root = debugfs_create_dir("mr_cache", mlx5_debugfs_get_dev_root(dev->mdev));
719
720 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
720 for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
721 ent = &cache->ent[i];
722 sprintf(ent->name, "%d", ent->order);
723 dir = debugfs_create_dir(ent->name, cache->root);
724 debugfs_create_file("size", 0600, dir, ent, &size_fops);
725 debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
726 debugfs_create_ulong("cur", 0400, dir, &ent->stored);
727 debugfs_create_u32("miss", 0600, dir, &ent->miss);
728 }
729}
730
731static void delay_time_func(struct timer_list *t)
732{
733 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
734
735 WRITE_ONCE(dev->fill_delay, 0);
736}
737
721 ent = &cache->ent[i];
722 sprintf(ent->name, "%d", ent->order);
723 dir = debugfs_create_dir(ent->name, cache->root);
724 debugfs_create_file("size", 0600, dir, ent, &size_fops);
725 debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
726 debugfs_create_ulong("cur", 0400, dir, &ent->stored);
727 debugfs_create_u32("miss", 0600, dir, &ent->miss);
728 }
729}
730
731static void delay_time_func(struct timer_list *t)
732{
733 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
734
735 WRITE_ONCE(dev->fill_delay, 0);
736}
737
738int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
738int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
739{
739{
740 struct mlx5_mr_cache *cache = &dev->cache;
740 struct mlx5_mkey_cache *cache = &dev->cache;
741 struct mlx5_cache_ent *ent;
742 int i;
743
744 mutex_init(&dev->slow_path_mutex);
745 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
746 if (!cache->wq) {
747 mlx5_ib_warn(dev, "failed to create work queue\n");
748 return -ENOMEM;
749 }
750
751 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
752 timer_setup(&dev->delay_timer, delay_time_func, 0);
741 struct mlx5_cache_ent *ent;
742 int i;
743
744 mutex_init(&dev->slow_path_mutex);
745 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
746 if (!cache->wq) {
747 mlx5_ib_warn(dev, "failed to create work queue\n");
748 return -ENOMEM;
749 }
750
751 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
752 timer_setup(&dev->delay_timer, delay_time_func, 0);
753 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
753 for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
754 ent = &cache->ent[i];
755 xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);
756 ent->order = i + 2;
757 ent->dev = dev;
758 ent->limit = 0;
759
760 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
761
754 ent = &cache->ent[i];
755 xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);
756 ent->order = i + 2;
757 ent->dev = dev;
758 ent->limit = 0;
759
760 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
761
762 if (i > MR_CACHE_LAST_STD_ENTRY) {
763 mlx5_odp_init_mr_cache_entry(ent);
762 if (i > MKEY_CACHE_LAST_STD_ENTRY) {
763 mlx5_odp_init_mkey_cache_entry(ent);
764 continue;
765 }
766
764 continue;
765 }
766
767 if (ent->order > mr_cache_max_order(dev))
767 if (ent->order > mkey_cache_max_order(dev))
768 continue;
769
770 ent->page = PAGE_SHIFT;
771 ent->ndescs = 1 << ent->order;
772 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
773 if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
774 !dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
775 mlx5r_umr_can_load_pas(dev, 0))
776 ent->limit = dev->mdev->profile.mr_cache[i].limit;
777 else
778 ent->limit = 0;
779 xa_lock_irq(&ent->mkeys);
780 queue_adjust_cache_locked(ent);
781 xa_unlock_irq(&ent->mkeys);
782 }
783
768 continue;
769
770 ent->page = PAGE_SHIFT;
771 ent->ndescs = 1 << ent->order;
772 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
773 if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) &&
774 !dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
775 mlx5r_umr_can_load_pas(dev, 0))
776 ent->limit = dev->mdev->profile.mr_cache[i].limit;
777 else
778 ent->limit = 0;
779 xa_lock_irq(&ent->mkeys);
780 queue_adjust_cache_locked(ent);
781 xa_unlock_irq(&ent->mkeys);
782 }
783
784 mlx5_mr_cache_debugfs_init(dev);
784 mlx5_mkey_cache_debugfs_init(dev);
785
786 return 0;
787}
788
785
786 return 0;
787}
788
789int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
789int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev)
790{
791 unsigned int i;
792
793 if (!dev->cache.wq)
794 return 0;
795
790{
791 unsigned int i;
792
793 if (!dev->cache.wq)
794 return 0;
795
796 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
796 for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {
797 struct mlx5_cache_ent *ent = &dev->cache.ent[i];
798
799 xa_lock_irq(&ent->mkeys);
800 ent->disabled = true;
801 xa_unlock_irq(&ent->mkeys);
802 cancel_delayed_work_sync(&ent->dwork);
803 }
804
797 struct mlx5_cache_ent *ent = &dev->cache.ent[i];
798
799 xa_lock_irq(&ent->mkeys);
800 ent->disabled = true;
801 xa_unlock_irq(&ent->mkeys);
802 cancel_delayed_work_sync(&ent->dwork);
803 }
804
805 mlx5_mr_cache_debugfs_cleanup(dev);
805 mlx5_mkey_cache_debugfs_cleanup(dev);
806 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
807
806 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
807
808 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
808 for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++)
809 clean_keys(dev, i);
810
811 destroy_workqueue(dev->cache.wq);
812 del_timer_sync(&dev->delay_timer);
813
814 return 0;
815}
816

--- 50 unchanged lines hidden (view full) ---

867 u64 offset;
868 int npages;
869
870 offset = addr & (page_size - 1);
871 npages = ALIGN(len + offset, page_size) >> page_shift;
872 return (npages + 1) / 2;
873}
874
809 clean_keys(dev, i);
810
811 destroy_workqueue(dev->cache.wq);
812 del_timer_sync(&dev->delay_timer);
813
814 return 0;
815}
816

--- 50 unchanged lines hidden (view full) ---

867 u64 offset;
868 int npages;
869
870 offset = addr & (page_size - 1);
871 npages = ALIGN(len + offset, page_size) >> page_shift;
872 return (npages + 1) / 2;
873}
874
875static int mr_cache_max_order(struct mlx5_ib_dev *dev)
875static int mkey_cache_max_order(struct mlx5_ib_dev *dev)
876{
877 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
876{
877 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
878 return MR_CACHE_LAST_STD_ENTRY + 2;
878 return MKEY_CACHE_LAST_STD_ENTRY + 2;
879 return MLX5_MAX_UMR_SHIFT;
880}
881
879 return MLX5_MAX_UMR_SHIFT;
880}
881
882static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
883 unsigned int order)
882static struct mlx5_cache_ent *mkey_cache_ent_from_order(struct mlx5_ib_dev *dev,
883 unsigned int order)
884{
884{
885 struct mlx5_mr_cache *cache = &dev->cache;
885 struct mlx5_mkey_cache *cache = &dev->cache;
886
887 if (order < cache->ent[0].order)
888 return &cache->ent[0];
889 order = order - cache->ent[0].order;
886
887 if (order < cache->ent[0].order)
888 return &cache->ent[0];
889 order = order - cache->ent[0].order;
890 if (order > MR_CACHE_LAST_STD_ENTRY)
890 if (order > MKEY_CACHE_LAST_STD_ENTRY)
891 return NULL;
892 return &cache->ent[order];
893}
894
895static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
896 u64 length, int access_flags, u64 iova)
897{
898 mr->ibmr.lkey = mr->mmkey.key;

--- 26 unchanged lines hidden (view full) ---

925
926 if (umem->is_dmabuf)
927 page_size = mlx5_umem_dmabuf_default_pgsz(umem, iova);
928 else
929 page_size = mlx5_umem_find_best_pgsz(umem, mkc, log_page_size,
930 0, iova);
931 if (WARN_ON(!page_size))
932 return ERR_PTR(-EINVAL);
891 return NULL;
892 return &cache->ent[order];
893}
894
895static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
896 u64 length, int access_flags, u64 iova)
897{
898 mr->ibmr.lkey = mr->mmkey.key;

--- 26 unchanged lines hidden (view full) ---

925
926 if (umem->is_dmabuf)
927 page_size = mlx5_umem_dmabuf_default_pgsz(umem, iova);
928 else
929 page_size = mlx5_umem_find_best_pgsz(umem, mkc, log_page_size,
930 0, iova);
931 if (WARN_ON(!page_size))
932 return ERR_PTR(-EINVAL);
933 ent = mr_cache_ent_from_order(
933 ent = mkey_cache_ent_from_order(
934 dev, order_base_2(ib_umem_num_dma_blocks(umem, page_size)));
935 /*
936 * Matches access in alloc_cache_mr(). If the MR can't come from the
937 * cache then synchronously create an uncached one.
938 */
939 if (!ent || ent->limit == 0 ||
940 !mlx5r_umr_can_reconfig(dev, 0, access_flags)) {
941 mutex_lock(&dev->slow_path_mutex);

--- 1403 unchanged lines hidden --- 		934 dev, order_base_2(ib_umem_num_dma_blocks(umem, page_size)));
935 /*
936 * Matches access in alloc_cache_mr(). If the MR can't come from the
937 * cache then synchronously create an uncached one.
938 */
939 if (!ent || ent->limit == 0 ||
940 !mlx5r_umr_can_reconfig(dev, 0, access_flags)) {
941 mutex_lock(&dev->slow_path_mutex);

--- 1403 unchanged lines hidden ---