1 /*
2  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *	   Redistribution and use in source and binary forms, with or
12  *	   without modification, are permitted provided that the following
13  *	   conditions are met:
14  *
15  *		- Redistributions of source code must retain the above
16  *		  copyright notice, this list of conditions and the following
17  *		  disclaimer.
18  *
19  *		- Redistributions in binary form must reproduce the above
20  *		  copyright notice, this list of conditions and the following
21  *		  disclaimer in the documentation and/or other materials
22  *		  provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include "rxe.h"
35 #include "rxe_loc.h"
36 
37 /* info about object pools
38  * note that mr and mw share a single index space
39  * so that one can map an lkey to the correct type of object
40  */
41 struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
42 	[RXE_TYPE_UC] = {
43 		.name		= "rxe-uc",
44 		.size		= sizeof(struct rxe_ucontext),
45 	},
46 	[RXE_TYPE_PD] = {
47 		.name		= "rxe-pd",
48 		.size		= sizeof(struct rxe_pd),
49 	},
50 	[RXE_TYPE_AH] = {
51 		.name		= "rxe-ah",
52 		.size		= sizeof(struct rxe_ah),
53 		.flags		= RXE_POOL_ATOMIC,
54 	},
55 	[RXE_TYPE_SRQ] = {
56 		.name		= "rxe-srq",
57 		.size		= sizeof(struct rxe_srq),
58 		.flags		= RXE_POOL_INDEX,
59 		.min_index	= RXE_MIN_SRQ_INDEX,
60 		.max_index	= RXE_MAX_SRQ_INDEX,
61 	},
62 	[RXE_TYPE_QP] = {
63 		.name		= "rxe-qp",
64 		.size		= sizeof(struct rxe_qp),
65 		.cleanup	= rxe_qp_cleanup,
66 		.flags		= RXE_POOL_INDEX,
67 		.min_index	= RXE_MIN_QP_INDEX,
68 		.max_index	= RXE_MAX_QP_INDEX,
69 	},
70 	[RXE_TYPE_CQ] = {
71 		.name		= "rxe-cq",
72 		.size		= sizeof(struct rxe_cq),
73 		.cleanup	= rxe_cq_cleanup,
74 	},
75 	[RXE_TYPE_MR] = {
76 		.name		= "rxe-mr",
77 		.size		= sizeof(struct rxe_mem),
78 		.cleanup	= rxe_mem_cleanup,
79 		.flags		= RXE_POOL_INDEX,
80 		.max_index	= RXE_MAX_MR_INDEX,
81 		.min_index	= RXE_MIN_MR_INDEX,
82 	},
83 	[RXE_TYPE_MW] = {
84 		.name		= "rxe-mw",
85 		.size		= sizeof(struct rxe_mem),
86 		.flags		= RXE_POOL_INDEX,
87 		.max_index	= RXE_MAX_MW_INDEX,
88 		.min_index	= RXE_MIN_MW_INDEX,
89 	},
90 	[RXE_TYPE_MC_GRP] = {
91 		.name		= "rxe-mc_grp",
92 		.size		= sizeof(struct rxe_mc_grp),
93 		.cleanup	= rxe_mc_cleanup,
94 		.flags		= RXE_POOL_KEY,
95 		.key_offset	= offsetof(struct rxe_mc_grp, mgid),
96 		.key_size	= sizeof(union ib_gid),
97 	},
98 	[RXE_TYPE_MC_ELEM] = {
99 		.name		= "rxe-mc_elem",
100 		.size		= sizeof(struct rxe_mc_elem),
101 		.flags		= RXE_POOL_ATOMIC,
102 	},
103 };
104 
105 static inline const char *pool_name(struct rxe_pool *pool)
106 {
107 	return rxe_type_info[pool->type].name;
108 }
109 
110 static inline struct kmem_cache *pool_cache(struct rxe_pool *pool)
111 {
112 	return rxe_type_info[pool->type].cache;
113 }
114 
115 int rxe_cache_init(void)
116 {
117 	int err;
118 	int i;
119 	size_t size;
120 	struct rxe_type_info *type;
121 
122 	for (i = 0; i < RXE_NUM_TYPES; i++) {
123 		type = &rxe_type_info[i];
124 		size = ALIGN(type->size, RXE_POOL_ALIGN);
125 		type->cache = kmem_cache_create(type->name, size,
126 				RXE_POOL_ALIGN,
127 				RXE_POOL_CACHE_FLAGS, NULL);
128 		if (!type->cache) {
129 			pr_err("Unable to init kmem cache for %s\n",
130 			       type->name);
131 			err = -ENOMEM;
132 			goto err1;
133 		}
134 	}
135 
136 	return 0;
137 
138 err1:
139 	while (--i >= 0) {
140 		kmem_cache_destroy(type->cache);
141 		type->cache = NULL;
142 	}
143 
144 	return err;
145 }
146 
147 void rxe_cache_exit(void)
148 {
149 	int i;
150 	struct rxe_type_info *type;
151 
152 	for (i = 0; i < RXE_NUM_TYPES; i++) {
153 		type = &rxe_type_info[i];
154 		kmem_cache_destroy(type->cache);
155 		type->cache = NULL;
156 	}
157 }
158 
159 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
160 {
161 	int err = 0;
162 	size_t size;
163 
164 	if ((max - min + 1) < pool->max_elem) {
165 		pr_warn("not enough indices for max_elem\n");
166 		err = -EINVAL;
167 		goto out;
168 	}
169 
170 	pool->max_index = max;
171 	pool->min_index = min;
172 
173 	size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
174 	pool->table = kmalloc(size, GFP_KERNEL);
175 	if (!pool->table) {
176 		err = -ENOMEM;
177 		goto out;
178 	}
179 
180 	pool->table_size = size;
181 	bitmap_zero(pool->table, max - min + 1);
182 
183 out:
184 	return err;
185 }
186 
187 int rxe_pool_init(
188 	struct rxe_dev		*rxe,
189 	struct rxe_pool		*pool,
190 	enum rxe_elem_type	type,
191 	unsigned int		max_elem)
192 {
193 	int			err = 0;
194 	size_t			size = rxe_type_info[type].size;
195 
196 	memset(pool, 0, sizeof(*pool));
197 
198 	pool->rxe		= rxe;
199 	pool->type		= type;
200 	pool->max_elem		= max_elem;
201 	pool->elem_size		= ALIGN(size, RXE_POOL_ALIGN);
202 	pool->flags		= rxe_type_info[type].flags;
203 	pool->tree		= RB_ROOT;
204 	pool->cleanup		= rxe_type_info[type].cleanup;
205 
206 	atomic_set(&pool->num_elem, 0);
207 
208 	kref_init(&pool->ref_cnt);
209 
210 	spin_lock_init(&pool->pool_lock);
211 
212 	if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
213 		err = rxe_pool_init_index(pool,
214 					  rxe_type_info[type].max_index,
215 					  rxe_type_info[type].min_index);
216 		if (err)
217 			goto out;
218 	}
219 
220 	if (rxe_type_info[type].flags & RXE_POOL_KEY) {
221 		pool->key_offset = rxe_type_info[type].key_offset;
222 		pool->key_size = rxe_type_info[type].key_size;
223 	}
224 
225 	pool->state = rxe_pool_valid;
226 
227 out:
228 	return err;
229 }
230 
231 static void rxe_pool_release(struct kref *kref)
232 {
233 	struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
234 
235 	pool->state = rxe_pool_invalid;
236 	kfree(pool->table);
237 }
238 
239 static void rxe_pool_put(struct rxe_pool *pool)
240 {
241 	kref_put(&pool->ref_cnt, rxe_pool_release);
242 }
243 
244 int rxe_pool_cleanup(struct rxe_pool *pool)
245 {
246 	unsigned long flags;
247 
248 	spin_lock_irqsave(&pool->pool_lock, flags);
249 	pool->state = rxe_pool_invalid;
250 	if (atomic_read(&pool->num_elem) > 0)
251 		pr_warn("%s pool destroyed with unfree'd elem\n",
252 			pool_name(pool));
253 	spin_unlock_irqrestore(&pool->pool_lock, flags);
254 
255 	rxe_pool_put(pool);
256 
257 	return 0;
258 }
259 
260 static u32 alloc_index(struct rxe_pool *pool)
261 {
262 	u32 index;
263 	u32 range = pool->max_index - pool->min_index + 1;
264 
265 	index = find_next_zero_bit(pool->table, range, pool->last);
266 	if (index >= range)
267 		index = find_first_zero_bit(pool->table, range);
268 
269 	WARN_ON_ONCE(index >= range);
270 	set_bit(index, pool->table);
271 	pool->last = index;
272 	return index + pool->min_index;
273 }
274 
275 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
276 {
277 	struct rb_node **link = &pool->tree.rb_node;
278 	struct rb_node *parent = NULL;
279 	struct rxe_pool_entry *elem;
280 
281 	while (*link) {
282 		parent = *link;
283 		elem = rb_entry(parent, struct rxe_pool_entry, node);
284 
285 		if (elem->index == new->index) {
286 			pr_warn("element already exists!\n");
287 			goto out;
288 		}
289 
290 		if (elem->index > new->index)
291 			link = &(*link)->rb_left;
292 		else
293 			link = &(*link)->rb_right;
294 	}
295 
296 	rb_link_node(&new->node, parent, link);
297 	rb_insert_color(&new->node, &pool->tree);
298 out:
299 	return;
300 }
301 
302 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
303 {
304 	struct rb_node **link = &pool->tree.rb_node;
305 	struct rb_node *parent = NULL;
306 	struct rxe_pool_entry *elem;
307 	int cmp;
308 
309 	while (*link) {
310 		parent = *link;
311 		elem = rb_entry(parent, struct rxe_pool_entry, node);
312 
313 		cmp = memcmp((u8 *)elem + pool->key_offset,
314 			     (u8 *)new + pool->key_offset, pool->key_size);
315 
316 		if (cmp == 0) {
317 			pr_warn("key already exists!\n");
318 			goto out;
319 		}
320 
321 		if (cmp > 0)
322 			link = &(*link)->rb_left;
323 		else
324 			link = &(*link)->rb_right;
325 	}
326 
327 	rb_link_node(&new->node, parent, link);
328 	rb_insert_color(&new->node, &pool->tree);
329 out:
330 	return;
331 }
332 
333 void rxe_add_key(void *arg, void *key)
334 {
335 	struct rxe_pool_entry *elem = arg;
336 	struct rxe_pool *pool = elem->pool;
337 	unsigned long flags;
338 
339 	spin_lock_irqsave(&pool->pool_lock, flags);
340 	memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
341 	insert_key(pool, elem);
342 	spin_unlock_irqrestore(&pool->pool_lock, flags);
343 }
344 
345 void rxe_drop_key(void *arg)
346 {
347 	struct rxe_pool_entry *elem = arg;
348 	struct rxe_pool *pool = elem->pool;
349 	unsigned long flags;
350 
351 	spin_lock_irqsave(&pool->pool_lock, flags);
352 	rb_erase(&elem->node, &pool->tree);
353 	spin_unlock_irqrestore(&pool->pool_lock, flags);
354 }
355 
356 void rxe_add_index(void *arg)
357 {
358 	struct rxe_pool_entry *elem = arg;
359 	struct rxe_pool *pool = elem->pool;
360 	unsigned long flags;
361 
362 	spin_lock_irqsave(&pool->pool_lock, flags);
363 	elem->index = alloc_index(pool);
364 	insert_index(pool, elem);
365 	spin_unlock_irqrestore(&pool->pool_lock, flags);
366 }
367 
368 void rxe_drop_index(void *arg)
369 {
370 	struct rxe_pool_entry *elem = arg;
371 	struct rxe_pool *pool = elem->pool;
372 	unsigned long flags;
373 
374 	spin_lock_irqsave(&pool->pool_lock, flags);
375 	clear_bit(elem->index - pool->min_index, pool->table);
376 	rb_erase(&elem->node, &pool->tree);
377 	spin_unlock_irqrestore(&pool->pool_lock, flags);
378 }
379 
380 void *rxe_alloc(struct rxe_pool *pool)
381 {
382 	struct rxe_pool_entry *elem;
383 	unsigned long flags;
384 
385 	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
386 
387 	spin_lock_irqsave(&pool->pool_lock, flags);
388 	if (pool->state != rxe_pool_valid) {
389 		spin_unlock_irqrestore(&pool->pool_lock, flags);
390 		return NULL;
391 	}
392 	kref_get(&pool->ref_cnt);
393 	spin_unlock_irqrestore(&pool->pool_lock, flags);
394 
395 	kref_get(&pool->rxe->ref_cnt);
396 
397 	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
398 		goto out_put_pool;
399 
400 	elem = kmem_cache_zalloc(pool_cache(pool),
401 				 (pool->flags & RXE_POOL_ATOMIC) ?
402 				 GFP_ATOMIC : GFP_KERNEL);
403 	if (!elem)
404 		goto out_put_pool;
405 
406 	elem->pool = pool;
407 	kref_init(&elem->ref_cnt);
408 
409 	return elem;
410 
411 out_put_pool:
412 	atomic_dec(&pool->num_elem);
413 	rxe_dev_put(pool->rxe);
414 	rxe_pool_put(pool);
415 	return NULL;
416 }
417 
418 void rxe_elem_release(struct kref *kref)
419 {
420 	struct rxe_pool_entry *elem =
421 		container_of(kref, struct rxe_pool_entry, ref_cnt);
422 	struct rxe_pool *pool = elem->pool;
423 
424 	if (pool->cleanup)
425 		pool->cleanup(elem);
426 
427 	kmem_cache_free(pool_cache(pool), elem);
428 	atomic_dec(&pool->num_elem);
429 	rxe_dev_put(pool->rxe);
430 	rxe_pool_put(pool);
431 }
432 
433 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
434 {
435 	struct rb_node *node = NULL;
436 	struct rxe_pool_entry *elem = NULL;
437 	unsigned long flags;
438 
439 	spin_lock_irqsave(&pool->pool_lock, flags);
440 
441 	if (pool->state != rxe_pool_valid)
442 		goto out;
443 
444 	node = pool->tree.rb_node;
445 
446 	while (node) {
447 		elem = rb_entry(node, struct rxe_pool_entry, node);
448 
449 		if (elem->index > index)
450 			node = node->rb_left;
451 		else if (elem->index < index)
452 			node = node->rb_right;
453 		else
454 			break;
455 	}
456 
457 	if (node)
458 		kref_get(&elem->ref_cnt);
459 
460 out:
461 	spin_unlock_irqrestore(&pool->pool_lock, flags);
462 	return node ? elem : NULL;
463 }
464 
465 void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
466 {
467 	struct rb_node *node = NULL;
468 	struct rxe_pool_entry *elem = NULL;
469 	int cmp;
470 	unsigned long flags;
471 
472 	spin_lock_irqsave(&pool->pool_lock, flags);
473 
474 	if (pool->state != rxe_pool_valid)
475 		goto out;
476 
477 	node = pool->tree.rb_node;
478 
479 	while (node) {
480 		elem = rb_entry(node, struct rxe_pool_entry, node);
481 
482 		cmp = memcmp((u8 *)elem + pool->key_offset,
483 			     key, pool->key_size);
484 
485 		if (cmp > 0)
486 			node = node->rb_left;
487 		else if (cmp < 0)
488 			node = node->rb_right;
489 		else
490 			break;
491 	}
492 
493 	if (node)
494 		kref_get(&elem->ref_cnt);
495 
496 out:
497 	spin_unlock_irqrestore(&pool->pool_lock, flags);
498 	return node ? elem : NULL;
499 }
500