1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5  */
6 
7 #include "rxe.h"
8 
9 #define RXE_POOL_TIMEOUT	(200)
10 #define RXE_POOL_ALIGN		(16)
11 
12 static const struct rxe_type_info {
13 	const char *name;
14 	size_t size;
15 	size_t elem_offset;
16 	void (*cleanup)(struct rxe_pool_elem *elem);
17 	u32 min_index;
18 	u32 max_index;
19 	u32 max_elem;
20 } rxe_type_info[RXE_NUM_TYPES] = {
21 	[RXE_TYPE_UC] = {
22 		.name		= "uc",
23 		.size		= sizeof(struct rxe_ucontext),
24 		.elem_offset	= offsetof(struct rxe_ucontext, elem),
25 		.min_index	= 1,
26 		.max_index	= RXE_MAX_UCONTEXT,
27 		.max_elem	= RXE_MAX_UCONTEXT,
28 	},
29 	[RXE_TYPE_PD] = {
30 		.name		= "pd",
31 		.size		= sizeof(struct rxe_pd),
32 		.elem_offset	= offsetof(struct rxe_pd, elem),
33 		.min_index	= 1,
34 		.max_index	= RXE_MAX_PD,
35 		.max_elem	= RXE_MAX_PD,
36 	},
37 	[RXE_TYPE_AH] = {
38 		.name		= "ah",
39 		.size		= sizeof(struct rxe_ah),
40 		.elem_offset	= offsetof(struct rxe_ah, elem),
41 		.min_index	= RXE_MIN_AH_INDEX,
42 		.max_index	= RXE_MAX_AH_INDEX,
43 		.max_elem	= RXE_MAX_AH,
44 	},
45 	[RXE_TYPE_SRQ] = {
46 		.name		= "srq",
47 		.size		= sizeof(struct rxe_srq),
48 		.elem_offset	= offsetof(struct rxe_srq, elem),
49 		.cleanup	= rxe_srq_cleanup,
50 		.min_index	= RXE_MIN_SRQ_INDEX,
51 		.max_index	= RXE_MAX_SRQ_INDEX,
52 		.max_elem	= RXE_MAX_SRQ,
53 	},
54 	[RXE_TYPE_QP] = {
55 		.name		= "qp",
56 		.size		= sizeof(struct rxe_qp),
57 		.elem_offset	= offsetof(struct rxe_qp, elem),
58 		.cleanup	= rxe_qp_cleanup,
59 		.min_index	= RXE_MIN_QP_INDEX,
60 		.max_index	= RXE_MAX_QP_INDEX,
61 		.max_elem	= RXE_MAX_QP,
62 	},
63 	[RXE_TYPE_CQ] = {
64 		.name		= "cq",
65 		.size		= sizeof(struct rxe_cq),
66 		.elem_offset	= offsetof(struct rxe_cq, elem),
67 		.cleanup	= rxe_cq_cleanup,
68 		.min_index	= 1,
69 		.max_index	= RXE_MAX_CQ,
70 		.max_elem	= RXE_MAX_CQ,
71 	},
72 	[RXE_TYPE_MR] = {
73 		.name		= "mr",
74 		.size		= sizeof(struct rxe_mr),
75 		.elem_offset	= offsetof(struct rxe_mr, elem),
76 		.cleanup	= rxe_mr_cleanup,
77 		.min_index	= RXE_MIN_MR_INDEX,
78 		.max_index	= RXE_MAX_MR_INDEX,
79 		.max_elem	= RXE_MAX_MR,
80 	},
81 	[RXE_TYPE_MW] = {
82 		.name		= "mw",
83 		.size		= sizeof(struct rxe_mw),
84 		.elem_offset	= offsetof(struct rxe_mw, elem),
85 		.cleanup	= rxe_mw_cleanup,
86 		.min_index	= RXE_MIN_MW_INDEX,
87 		.max_index	= RXE_MAX_MW_INDEX,
88 		.max_elem	= RXE_MAX_MW,
89 	},
90 };
91 
92 void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
93 		   enum rxe_elem_type type)
94 {
95 	const struct rxe_type_info *info = &rxe_type_info[type];
96 
97 	memset(pool, 0, sizeof(*pool));
98 
99 	pool->rxe		= rxe;
100 	pool->name		= info->name;
101 	pool->type		= type;
102 	pool->max_elem		= info->max_elem;
103 	pool->elem_size		= ALIGN(info->size, RXE_POOL_ALIGN);
104 	pool->elem_offset	= info->elem_offset;
105 	pool->cleanup		= info->cleanup;
106 
107 	atomic_set(&pool->num_elem, 0);
108 
109 	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
110 	pool->limit.min = info->min_index;
111 	pool->limit.max = info->max_index;
112 }
113 
114 void rxe_pool_cleanup(struct rxe_pool *pool)
115 {
116 	WARN_ON(!xa_empty(&pool->xa));
117 }
118 
119 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem,
120 				bool sleepable)
121 {
122 	int err;
123 	gfp_t gfp_flags;
124 
125 	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
126 		goto err_cnt;
127 
128 	elem->pool = pool;
129 	elem->obj = (u8 *)elem - pool->elem_offset;
130 	kref_init(&elem->ref_cnt);
131 	init_completion(&elem->complete);
132 
133 	/* AH objects are unique in that the create_ah verb
134 	 * can be called in atomic context. If the create_ah
135 	 * call is not sleepable use GFP_ATOMIC.
136 	 */
137 	gfp_flags = sleepable ? GFP_KERNEL : GFP_ATOMIC;
138 
139 	if (sleepable)
140 		might_sleep();
141 	err = xa_alloc_cyclic(&pool->xa, &elem->index, NULL, pool->limit,
142 			      &pool->next, gfp_flags);
143 	if (err < 0)
144 		goto err_cnt;
145 
146 	return 0;
147 
148 err_cnt:
149 	atomic_dec(&pool->num_elem);
150 	return -EINVAL;
151 }
152 
153 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
154 {
155 	struct rxe_pool_elem *elem;
156 	struct xarray *xa = &pool->xa;
157 	void *obj;
158 
159 	rcu_read_lock();
160 	elem = xa_load(xa, index);
161 	if (elem && kref_get_unless_zero(&elem->ref_cnt))
162 		obj = elem->obj;
163 	else
164 		obj = NULL;
165 	rcu_read_unlock();
166 
167 	return obj;
168 }
169 
170 static void rxe_elem_release(struct kref *kref)
171 {
172 	struct rxe_pool_elem *elem = container_of(kref, typeof(*elem), ref_cnt);
173 
174 	complete(&elem->complete);
175 }
176 
177 int __rxe_cleanup(struct rxe_pool_elem *elem, bool sleepable)
178 {
179 	struct rxe_pool *pool = elem->pool;
180 	struct xarray *xa = &pool->xa;
181 	static int timeout = RXE_POOL_TIMEOUT;
182 	int ret, err = 0;
183 	void *xa_ret;
184 
185 	if (sleepable)
186 		might_sleep();
187 
188 	/* erase xarray entry to prevent looking up
189 	 * the pool elem from its index
190 	 */
191 	xa_ret = xa_erase(xa, elem->index);
192 	WARN_ON(xa_err(xa_ret));
193 
194 	/* if this is the last call to rxe_put complete the
195 	 * object. It is safe to touch obj->elem after this since
196 	 * it is freed below
197 	 */
198 	__rxe_put(elem);
199 
200 	/* wait until all references to the object have been
201 	 * dropped before final object specific cleanup and
202 	 * return to rdma-core
203 	 */
204 	if (sleepable) {
205 		if (!completion_done(&elem->complete) && timeout) {
206 			ret = wait_for_completion_timeout(&elem->complete,
207 					timeout);
208 
209 			/* Shouldn't happen. There are still references to
210 			 * the object but, rather than deadlock, free the
211 			 * object or pass back to rdma-core.
212 			 */
213 			if (WARN_ON(!ret))
214 				err = -EINVAL;
215 		}
216 	} else {
217 		unsigned long until = jiffies + timeout;
218 
219 		/* AH objects are unique in that the destroy_ah verb
220 		 * can be called in atomic context. This delay
221 		 * replaces the wait_for_completion call above
222 		 * when the destroy_ah call is not sleepable
223 		 */
224 		while (!completion_done(&elem->complete) &&
225 				time_before(jiffies, until))
226 			mdelay(1);
227 
228 		if (WARN_ON(!completion_done(&elem->complete)))
229 			err = -EINVAL;
230 	}
231 
232 	if (pool->cleanup)
233 		pool->cleanup(elem);
234 
235 	atomic_dec(&pool->num_elem);
236 
237 	return err;
238 }
239 
240 int __rxe_get(struct rxe_pool_elem *elem)
241 {
242 	return kref_get_unless_zero(&elem->ref_cnt);
243 }
244 
245 int __rxe_put(struct rxe_pool_elem *elem)
246 {
247 	return kref_put(&elem->ref_cnt, rxe_elem_release);
248 }
249 
250 void __rxe_finalize(struct rxe_pool_elem *elem)
251 {
252 	void *xa_ret;
253 
254 	xa_ret = xa_store(&elem->pool->xa, elem->index, elem, GFP_KERNEL);
255 	WARN_ON(xa_err(xa_ret));
256 }
257