180 					WRITE_ONCE(ent->dev->cache.last_add,  in push_mkey_locked()
274 	WRITE_ONCE(dev->cache.last_add, jiffies);  in create_mkey_callback()
378 	WRITE_ONCE(ent->dev->cache.last_add, jiffies);  in create_cache_mkey()
533 static bool someone_adding(struct mlx5_mkey_cache *cache)  in someone_adding()  argument
539 	mutex_lock(&cache->rb_lock);  in someone_adding()
540 	for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) {  in someone_adding()
546 			mutex_unlock(&cache->rb_lock);  in someone_adding()
550 	mutex_unlock(&cache->rb_lock);  in someone_adding()
567 		mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);  in queue_adjust_cache_locked()
574 		mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);  in queue_adjust_cache_locked()
581 			queue_delayed_work(ent->dev->cache.wq, &ent->dwork,  in queue_adjust_cache_locked()
584 			mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0);  in queue_adjust_cache_locked()
591 	struct mlx5_mkey_cache *cache = &dev->cache;  in __cache_work_func()  local
616 				queue_delayed_work(cache->wq, &ent->dwork,  in __cache_work_func()
636 		need_delay = need_resched() || someone_adding(cache) ||  in __cache_work_func()
638 					 READ_ONCE(cache->last_add) + 300 * HZ);  in __cache_work_func()
643 			queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);  in __cache_work_func()
686 static int mlx5_cache_ent_insert(struct mlx5_mkey_cache *cache,  in mlx5_cache_ent_insert()  argument
689 	struct rb_node **new = &cache->rb_root.rb_node, *parent = NULL;  in mlx5_cache_ent_insert()
708 	rb_insert_color(&ent->node, &cache->rb_root);  in mlx5_cache_ent_insert()
717 	struct rb_node *node = dev->cache.rb_root.rb_node;  in mkey_cache_ent_from_rb_key()
852 	debugfs_remove_recursive(dev->cache.fs_root);  in mlx5_mkey_cache_debugfs_cleanup()
853 	dev->cache.fs_root = NULL;  in mlx5_mkey_cache_debugfs_cleanup()
869 	dir = debugfs_create_dir(ent->name, dev->cache.fs_root);  in mlx5_mkey_cache_debugfs_add_ent()
879 	struct mlx5_mkey_cache *cache = &dev->cache;  in mlx5_mkey_cache_debugfs_init()  local
884 	cache->fs_root = debugfs_create_dir("mr_cache", dbg_root);  in mlx5_mkey_cache_debugfs_init()
914 	ret = mlx5_cache_ent_insert(&dev->cache, ent);  in mlx5r_cache_create_ent_locked()
935 		mod_delayed_work(ent->dev->cache.wq,  in mlx5r_cache_create_ent_locked()
936 				 &ent->dev->cache.remove_ent_dwork,  in mlx5r_cache_create_ent_locked()
945 	struct mlx5_mkey_cache *cache;  in remove_ent_work_func()  local
949 	cache = container_of(work, struct mlx5_mkey_cache,  in remove_ent_work_func()
951 	mutex_lock(&cache->rb_lock);  in remove_ent_work_func()
952 	cur = rb_last(&cache->rb_root);  in remove_ent_work_func()
956 		mutex_unlock(&cache->rb_lock);  in remove_ent_work_func()
961 			mutex_lock(&cache->rb_lock);  in remove_ent_work_func()
967 		mutex_lock(&cache->rb_lock);  in remove_ent_work_func()
969 	mutex_unlock(&cache->rb_lock);  in remove_ent_work_func()
974 	struct mlx5_mkey_cache *cache = &dev->cache;  in mlx5_mkey_cache_init()  local
975 	struct rb_root *root = &dev->cache.rb_root;  in mlx5_mkey_cache_init()
985 	mutex_init(&dev->cache.rb_lock);  in mlx5_mkey_cache_init()
986 	dev->cache.rb_root = RB_ROOT;  in mlx5_mkey_cache_init()
987 	INIT_DELAYED_WORK(&dev->cache.remove_ent_dwork, remove_ent_work_func);  in mlx5_mkey_cache_init()
988 	cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);  in mlx5_mkey_cache_init()
989 	if (!cache->wq) {  in mlx5_mkey_cache_init()
997 	mutex_lock(&cache->rb_lock);  in mlx5_mkey_cache_init()
1011 	mutex_unlock(&cache->rb_lock);  in mlx5_mkey_cache_init()
1022 	mutex_unlock(&cache->rb_lock);  in mlx5_mkey_cache_init()
1030 	struct rb_root *root = &dev->cache.rb_root;  in mlx5_mkey_cache_cleanup()
1034 	if (!dev->cache.wq)  in mlx5_mkey_cache_cleanup()
1037 	mutex_lock(&dev->cache.rb_lock);  in mlx5_mkey_cache_cleanup()
1038 	cancel_delayed_work(&dev->cache.remove_ent_dwork);  in mlx5_mkey_cache_cleanup()
1046 	mutex_unlock(&dev->cache.rb_lock);  in mlx5_mkey_cache_cleanup()
1052 	flush_workqueue(dev->cache.wq);  in mlx5_mkey_cache_cleanup()
1058 	mutex_lock(&dev->cache.rb_lock);  in mlx5_mkey_cache_cleanup()
1067 	mutex_unlock(&dev->cache.rb_lock);  in mlx5_mkey_cache_cleanup()
1069 	destroy_workqueue(dev->cache.wq);  in mlx5_mkey_cache_cleanup()
1833 	struct mlx5_mkey_cache *cache = &dev->cache;  in cache_ent_find_and_store()  local
1843 	mutex_lock(&cache->rb_lock);  in cache_ent_find_and_store()
1848 				mutex_unlock(&cache->rb_lock);  in cache_ent_find_and_store()
1853 			mutex_unlock(&cache->rb_lock);  in cache_ent_find_and_store()
1859 	mutex_unlock(&cache->rb_lock);  in cache_ent_find_and_store()