1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5  */
6 
7 #include "rxe.h"
8 
9 #define RXE_POOL_ALIGN		(16)
10 
11 static const struct rxe_type_info {
12 	const char *name;
13 	size_t size;
14 	size_t elem_offset;
15 	void (*cleanup)(struct rxe_pool_elem *obj);
16 	enum rxe_pool_flags flags;
17 	u32 min_index;
18 	u32 max_index;
19 } rxe_type_info[RXE_NUM_TYPES] = {
20 	[RXE_TYPE_UC] = {
21 		.name		= "rxe-uc",
22 		.size		= sizeof(struct rxe_ucontext),
23 		.elem_offset	= offsetof(struct rxe_ucontext, elem),
24 		.flags          = RXE_POOL_NO_ALLOC,
25 	},
26 	[RXE_TYPE_PD] = {
27 		.name		= "rxe-pd",
28 		.size		= sizeof(struct rxe_pd),
29 		.elem_offset	= offsetof(struct rxe_pd, elem),
30 		.flags		= RXE_POOL_NO_ALLOC,
31 	},
32 	[RXE_TYPE_AH] = {
33 		.name		= "rxe-ah",
34 		.size		= sizeof(struct rxe_ah),
35 		.elem_offset	= offsetof(struct rxe_ah, elem),
36 		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
37 		.min_index	= RXE_MIN_AH_INDEX,
38 		.max_index	= RXE_MAX_AH_INDEX,
39 	},
40 	[RXE_TYPE_SRQ] = {
41 		.name		= "rxe-srq",
42 		.size		= sizeof(struct rxe_srq),
43 		.elem_offset	= offsetof(struct rxe_srq, elem),
44 		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
45 		.min_index	= RXE_MIN_SRQ_INDEX,
46 		.max_index	= RXE_MAX_SRQ_INDEX,
47 	},
48 	[RXE_TYPE_QP] = {
49 		.name		= "rxe-qp",
50 		.size		= sizeof(struct rxe_qp),
51 		.elem_offset	= offsetof(struct rxe_qp, elem),
52 		.cleanup	= rxe_qp_cleanup,
53 		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
54 		.min_index	= RXE_MIN_QP_INDEX,
55 		.max_index	= RXE_MAX_QP_INDEX,
56 	},
57 	[RXE_TYPE_CQ] = {
58 		.name		= "rxe-cq",
59 		.size		= sizeof(struct rxe_cq),
60 		.elem_offset	= offsetof(struct rxe_cq, elem),
61 		.flags          = RXE_POOL_NO_ALLOC,
62 		.cleanup	= rxe_cq_cleanup,
63 	},
64 	[RXE_TYPE_MR] = {
65 		.name		= "rxe-mr",
66 		.size		= sizeof(struct rxe_mr),
67 		.elem_offset	= offsetof(struct rxe_mr, elem),
68 		.cleanup	= rxe_mr_cleanup,
69 		.flags		= RXE_POOL_INDEX,
70 		.min_index	= RXE_MIN_MR_INDEX,
71 		.max_index	= RXE_MAX_MR_INDEX,
72 	},
73 	[RXE_TYPE_MW] = {
74 		.name		= "rxe-mw",
75 		.size		= sizeof(struct rxe_mw),
76 		.elem_offset	= offsetof(struct rxe_mw, elem),
77 		.cleanup	= rxe_mw_cleanup,
78 		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
79 		.min_index	= RXE_MIN_MW_INDEX,
80 		.max_index	= RXE_MAX_MW_INDEX,
81 	},
82 	[RXE_TYPE_MC_GRP] = {
83 		.name		= "rxe-mc_grp",
84 		.size		= sizeof(struct rxe_mcg),
85 		.elem_offset	= offsetof(struct rxe_mcg, elem),
86 		.cleanup	= rxe_mc_cleanup,
87 		.flags		= RXE_POOL_KEY,
88 		.key_offset	= offsetof(struct rxe_mcg, mgid),
89 		.key_size	= sizeof(union ib_gid),
90 	},
91 };
92 
93 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
94 {
95 	int err = 0;
96 
97 	if ((max - min + 1) < pool->max_elem) {
98 		pr_warn("not enough indices for max_elem\n");
99 		err = -EINVAL;
100 		goto out;
101 	}
102 
103 	pool->index.max_index = max;
104 	pool->index.min_index = min;
105 
106 	pool->index.table = bitmap_zalloc(max - min + 1, GFP_KERNEL);
107 	if (!pool->index.table) {
108 		err = -ENOMEM;
109 		goto out;
110 	}
111 
112 out:
113 	return err;
114 }
115 
116 int rxe_pool_init(
117 	struct rxe_dev		*rxe,
118 	struct rxe_pool		*pool,
119 	enum rxe_elem_type	type,
120 	unsigned int		max_elem)
121 {
122 	const struct rxe_type_info *info = &rxe_type_info[type];
123 	int			err = 0;
124 
125 	memset(pool, 0, sizeof(*pool));
126 
127 	pool->rxe		= rxe;
128 	pool->name		= info->name;
129 	pool->type		= type;
130 	pool->max_elem		= max_elem;
131 	pool->elem_size		= ALIGN(info->size, RXE_POOL_ALIGN);
132 	pool->elem_offset	= info->elem_offset;
133 	pool->flags		= info->flags;
134 	pool->cleanup		= info->cleanup;
135 
136 	atomic_set(&pool->num_elem, 0);
137 
138 	rwlock_init(&pool->pool_lock);
139 
140 	if (pool->flags & RXE_POOL_INDEX) {
141 		pool->index.tree = RB_ROOT;
142 		err = rxe_pool_init_index(pool, info->max_index,
143 					  info->min_index);
144 		if (err)
145 			goto out;
146 	}
147 
148 out:
149 	return err;
150 }
151 
152 void rxe_pool_cleanup(struct rxe_pool *pool)
153 {
154 	if (atomic_read(&pool->num_elem) > 0)
155 		pr_warn("%s pool destroyed with unfree'd elem\n",
156 			pool->name);
157 
158 	if (pool->flags & RXE_POOL_INDEX)
159 		bitmap_free(pool->index.table);
160 }
161 
162 static u32 alloc_index(struct rxe_pool *pool)
163 {
164 	u32 index;
165 	u32 range = pool->index.max_index - pool->index.min_index + 1;
166 
167 	index = find_next_zero_bit(pool->index.table, range, pool->index.last);
168 	if (index >= range)
169 		index = find_first_zero_bit(pool->index.table, range);
170 
171 	WARN_ON_ONCE(index >= range);
172 	set_bit(index, pool->index.table);
173 	pool->index.last = index;
174 	return index + pool->index.min_index;
175 }
176 
177 static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_elem *new)
178 {
179 	struct rb_node **link = &pool->index.tree.rb_node;
180 	struct rb_node *parent = NULL;
181 	struct rxe_pool_elem *elem;
182 
183 	while (*link) {
184 		parent = *link;
185 		elem = rb_entry(parent, struct rxe_pool_elem, index_node);
186 
187 		if (elem->index == new->index) {
188 			pr_warn("element already exists!\n");
189 			return -EINVAL;
190 		}
191 
192 		if (elem->index > new->index)
193 			link = &(*link)->rb_left;
194 		else
195 			link = &(*link)->rb_right;
196 	}
197 
198 	rb_link_node(&new->index_node, parent, link);
199 	rb_insert_color(&new->index_node, &pool->index.tree);
200 
201 	return 0;
202 }
203 
204 int __rxe_add_index_locked(struct rxe_pool_elem *elem)
205 {
206 	struct rxe_pool *pool = elem->pool;
207 	int err;
208 
209 	elem->index = alloc_index(pool);
210 	err = rxe_insert_index(pool, elem);
211 
212 	return err;
213 }
214 
215 int __rxe_add_index(struct rxe_pool_elem *elem)
216 {
217 	struct rxe_pool *pool = elem->pool;
218 	unsigned long flags;
219 	int err;
220 
221 	write_lock_irqsave(&pool->pool_lock, flags);
222 	err = __rxe_add_index_locked(elem);
223 	write_unlock_irqrestore(&pool->pool_lock, flags);
224 
225 	return err;
226 }
227 
228 void __rxe_drop_index_locked(struct rxe_pool_elem *elem)
229 {
230 	struct rxe_pool *pool = elem->pool;
231 
232 	clear_bit(elem->index - pool->index.min_index, pool->index.table);
233 	rb_erase(&elem->index_node, &pool->index.tree);
234 }
235 
236 void __rxe_drop_index(struct rxe_pool_elem *elem)
237 {
238 	struct rxe_pool *pool = elem->pool;
239 	unsigned long flags;
240 
241 	write_lock_irqsave(&pool->pool_lock, flags);
242 	__rxe_drop_index_locked(elem);
243 	write_unlock_irqrestore(&pool->pool_lock, flags);
244 }
245 
246 void *rxe_alloc_locked(struct rxe_pool *pool)
247 {
248 	struct rxe_pool_elem *elem;
249 	void *obj;
250 
251 	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
252 		goto out_cnt;
253 
254 	obj = kzalloc(pool->elem_size, GFP_ATOMIC);
255 	if (!obj)
256 		goto out_cnt;
257 
258 	elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset);
259 
260 	elem->pool = pool;
261 	elem->obj = obj;
262 	kref_init(&elem->ref_cnt);
263 
264 	return obj;
265 
266 out_cnt:
267 	atomic_dec(&pool->num_elem);
268 	return NULL;
269 }
270 
271 void *rxe_alloc(struct rxe_pool *pool)
272 {
273 	struct rxe_pool_elem *elem;
274 	void *obj;
275 
276 	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
277 		goto out_cnt;
278 
279 	obj = kzalloc(pool->elem_size, GFP_KERNEL);
280 	if (!obj)
281 		goto out_cnt;
282 
283 	elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset);
284 
285 	elem->pool = pool;
286 	elem->obj = obj;
287 	kref_init(&elem->ref_cnt);
288 
289 	return obj;
290 
291 out_cnt:
292 	atomic_dec(&pool->num_elem);
293 	return NULL;
294 }
295 
296 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
297 {
298 	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
299 		goto out_cnt;
300 
301 	elem->pool = pool;
302 	elem->obj = (u8 *)elem - pool->elem_offset;
303 	kref_init(&elem->ref_cnt);
304 
305 	return 0;
306 
307 out_cnt:
308 	atomic_dec(&pool->num_elem);
309 	return -EINVAL;
310 }
311 
312 void rxe_elem_release(struct kref *kref)
313 {
314 	struct rxe_pool_elem *elem =
315 		container_of(kref, struct rxe_pool_elem, ref_cnt);
316 	struct rxe_pool *pool = elem->pool;
317 	void *obj;
318 
319 	if (pool->cleanup)
320 		pool->cleanup(elem);
321 
322 	if (!(pool->flags & RXE_POOL_NO_ALLOC)) {
323 		obj = elem->obj;
324 		kfree(obj);
325 	}
326 
327 	atomic_dec(&pool->num_elem);
328 }
329 
330 void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)
331 {
332 	struct rb_node *node;
333 	struct rxe_pool_elem *elem;
334 	void *obj;
335 
336 	node = pool->index.tree.rb_node;
337 
338 	while (node) {
339 		elem = rb_entry(node, struct rxe_pool_elem, index_node);
340 
341 		if (elem->index > index)
342 			node = node->rb_left;
343 		else if (elem->index < index)
344 			node = node->rb_right;
345 		else
346 			break;
347 	}
348 
349 	if (node) {
350 		kref_get(&elem->ref_cnt);
351 		obj = elem->obj;
352 	} else {
353 		obj = NULL;
354 	}
355 
356 	return obj;
357 }
358 
359 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
360 {
361 	unsigned long flags;
362 	void *obj;
363 
364 	read_lock_irqsave(&pool->pool_lock, flags);
365 	obj = rxe_pool_get_index_locked(pool, index);
366 	read_unlock_irqrestore(&pool->pool_lock, flags);
367 
368 	return obj;
369 }
370