1 /*
2  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *	   Redistribution and use in source and binary forms, with or
12  *	   without modification, are permitted provided that the following
13  *	   conditions are met:
14  *
15  *		- Redistributions of source code must retain the above
16  *		  copyright notice, this list of conditions and the following
17  *		  disclaimer.
18  *
19  *		- Redistributions in binary form must reproduce the above
20  *		  copyright notice, this list of conditions and the following
21  *		  disclaimer in the documentation and/or other materials
22  *		  provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include "rxe.h"
35 #include "rxe_loc.h"
36 
37 /* info about object pools
38  * note that mr and mw share a single index space
39  * so that one can map an lkey to the correct type of object
40  */
41 struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
42 	[RXE_TYPE_UC] = {
43 		.name		= "rxe-uc",
44 		.size		= sizeof(struct rxe_ucontext),
45 	},
46 	[RXE_TYPE_PD] = {
47 		.name		= "rxe-pd",
48 		.size		= sizeof(struct rxe_pd),
49 	},
50 	[RXE_TYPE_AH] = {
51 		.name		= "rxe-ah",
52 		.size		= sizeof(struct rxe_ah),
53 		.flags		= RXE_POOL_ATOMIC,
54 	},
55 	[RXE_TYPE_SRQ] = {
56 		.name		= "rxe-srq",
57 		.size		= sizeof(struct rxe_srq),
58 		.flags		= RXE_POOL_INDEX,
59 		.min_index	= RXE_MIN_SRQ_INDEX,
60 		.max_index	= RXE_MAX_SRQ_INDEX,
61 	},
62 	[RXE_TYPE_QP] = {
63 		.name		= "rxe-qp",
64 		.size		= sizeof(struct rxe_qp),
65 		.cleanup	= rxe_qp_cleanup,
66 		.flags		= RXE_POOL_INDEX,
67 		.min_index	= RXE_MIN_QP_INDEX,
68 		.max_index	= RXE_MAX_QP_INDEX,
69 	},
70 	[RXE_TYPE_CQ] = {
71 		.name		= "rxe-cq",
72 		.size		= sizeof(struct rxe_cq),
73 		.cleanup	= rxe_cq_cleanup,
74 	},
75 	[RXE_TYPE_MR] = {
76 		.name		= "rxe-mr",
77 		.size		= sizeof(struct rxe_mem),
78 		.cleanup	= rxe_mem_cleanup,
79 		.flags		= RXE_POOL_INDEX,
80 		.max_index	= RXE_MAX_MR_INDEX,
81 		.min_index	= RXE_MIN_MR_INDEX,
82 	},
83 	[RXE_TYPE_MW] = {
84 		.name		= "rxe-mw",
85 		.size		= sizeof(struct rxe_mem),
86 		.flags		= RXE_POOL_INDEX,
87 		.max_index	= RXE_MAX_MW_INDEX,
88 		.min_index	= RXE_MIN_MW_INDEX,
89 	},
90 	[RXE_TYPE_MC_GRP] = {
91 		.name		= "rxe-mc_grp",
92 		.size		= sizeof(struct rxe_mc_grp),
93 		.cleanup	= rxe_mc_cleanup,
94 		.flags		= RXE_POOL_KEY,
95 		.key_offset	= offsetof(struct rxe_mc_grp, mgid),
96 		.key_size	= sizeof(union ib_gid),
97 	},
98 	[RXE_TYPE_MC_ELEM] = {
99 		.name		= "rxe-mc_elem",
100 		.size		= sizeof(struct rxe_mc_elem),
101 		.flags		= RXE_POOL_ATOMIC,
102 	},
103 };
104 
105 static inline const char *pool_name(struct rxe_pool *pool)
106 {
107 	return rxe_type_info[pool->type].name;
108 }
109 
110 static inline struct kmem_cache *pool_cache(struct rxe_pool *pool)
111 {
112 	return rxe_type_info[pool->type].cache;
113 }
114 
115 static void rxe_cache_clean(size_t cnt)
116 {
117 	int i;
118 	struct rxe_type_info *type;
119 
120 	for (i = 0; i < cnt; i++) {
121 		type = &rxe_type_info[i];
122 		kmem_cache_destroy(type->cache);
123 		type->cache = NULL;
124 	}
125 }
126 
127 int rxe_cache_init(void)
128 {
129 	int err;
130 	int i;
131 	size_t size;
132 	struct rxe_type_info *type;
133 
134 	for (i = 0; i < RXE_NUM_TYPES; i++) {
135 		type = &rxe_type_info[i];
136 		size = ALIGN(type->size, RXE_POOL_ALIGN);
137 		type->cache = kmem_cache_create(type->name, size,
138 				RXE_POOL_ALIGN,
139 				RXE_POOL_CACHE_FLAGS, NULL);
140 		if (!type->cache) {
141 			pr_err("Unable to init kmem cache for %s\n",
142 			       type->name);
143 			err = -ENOMEM;
144 			goto err1;
145 		}
146 	}
147 
148 	return 0;
149 
150 err1:
151 	rxe_cache_clean(i);
152 
153 	return err;
154 }
155 
156 void rxe_cache_exit(void)
157 {
158 	rxe_cache_clean(RXE_NUM_TYPES);
159 }
160 
161 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
162 {
163 	int err = 0;
164 	size_t size;
165 
166 	if ((max - min + 1) < pool->max_elem) {
167 		pr_warn("not enough indices for max_elem\n");
168 		err = -EINVAL;
169 		goto out;
170 	}
171 
172 	pool->max_index = max;
173 	pool->min_index = min;
174 
175 	size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
176 	pool->table = kmalloc(size, GFP_KERNEL);
177 	if (!pool->table) {
178 		err = -ENOMEM;
179 		goto out;
180 	}
181 
182 	pool->table_size = size;
183 	bitmap_zero(pool->table, max - min + 1);
184 
185 out:
186 	return err;
187 }
188 
189 int rxe_pool_init(
190 	struct rxe_dev		*rxe,
191 	struct rxe_pool		*pool,
192 	enum rxe_elem_type	type,
193 	unsigned int		max_elem)
194 {
195 	int			err = 0;
196 	size_t			size = rxe_type_info[type].size;
197 
198 	memset(pool, 0, sizeof(*pool));
199 
200 	pool->rxe		= rxe;
201 	pool->type		= type;
202 	pool->max_elem		= max_elem;
203 	pool->elem_size		= ALIGN(size, RXE_POOL_ALIGN);
204 	pool->flags		= rxe_type_info[type].flags;
205 	pool->tree		= RB_ROOT;
206 	pool->cleanup		= rxe_type_info[type].cleanup;
207 
208 	atomic_set(&pool->num_elem, 0);
209 
210 	kref_init(&pool->ref_cnt);
211 
212 	rwlock_init(&pool->pool_lock);
213 
214 	if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
215 		err = rxe_pool_init_index(pool,
216 					  rxe_type_info[type].max_index,
217 					  rxe_type_info[type].min_index);
218 		if (err)
219 			goto out;
220 	}
221 
222 	if (rxe_type_info[type].flags & RXE_POOL_KEY) {
223 		pool->key_offset = rxe_type_info[type].key_offset;
224 		pool->key_size = rxe_type_info[type].key_size;
225 	}
226 
227 	pool->state = RXE_POOL_STATE_VALID;
228 
229 out:
230 	return err;
231 }
232 
233 static void rxe_pool_release(struct kref *kref)
234 {
235 	struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
236 
237 	pool->state = RXE_POOL_STATE_INVALID;
238 	kfree(pool->table);
239 }
240 
241 static void rxe_pool_put(struct rxe_pool *pool)
242 {
243 	kref_put(&pool->ref_cnt, rxe_pool_release);
244 }
245 
246 void rxe_pool_cleanup(struct rxe_pool *pool)
247 {
248 	unsigned long flags;
249 
250 	write_lock_irqsave(&pool->pool_lock, flags);
251 	pool->state = RXE_POOL_STATE_INVALID;
252 	if (atomic_read(&pool->num_elem) > 0)
253 		pr_warn("%s pool destroyed with unfree'd elem\n",
254 			pool_name(pool));
255 	write_unlock_irqrestore(&pool->pool_lock, flags);
256 
257 	rxe_pool_put(pool);
258 }
259 
260 static u32 alloc_index(struct rxe_pool *pool)
261 {
262 	u32 index;
263 	u32 range = pool->max_index - pool->min_index + 1;
264 
265 	index = find_next_zero_bit(pool->table, range, pool->last);
266 	if (index >= range)
267 		index = find_first_zero_bit(pool->table, range);
268 
269 	WARN_ON_ONCE(index >= range);
270 	set_bit(index, pool->table);
271 	pool->last = index;
272 	return index + pool->min_index;
273 }
274 
275 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
276 {
277 	struct rb_node **link = &pool->tree.rb_node;
278 	struct rb_node *parent = NULL;
279 	struct rxe_pool_entry *elem;
280 
281 	while (*link) {
282 		parent = *link;
283 		elem = rb_entry(parent, struct rxe_pool_entry, node);
284 
285 		if (elem->index == new->index) {
286 			pr_warn("element already exists!\n");
287 			goto out;
288 		}
289 
290 		if (elem->index > new->index)
291 			link = &(*link)->rb_left;
292 		else
293 			link = &(*link)->rb_right;
294 	}
295 
296 	rb_link_node(&new->node, parent, link);
297 	rb_insert_color(&new->node, &pool->tree);
298 out:
299 	return;
300 }
301 
302 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
303 {
304 	struct rb_node **link = &pool->tree.rb_node;
305 	struct rb_node *parent = NULL;
306 	struct rxe_pool_entry *elem;
307 	int cmp;
308 
309 	while (*link) {
310 		parent = *link;
311 		elem = rb_entry(parent, struct rxe_pool_entry, node);
312 
313 		cmp = memcmp((u8 *)elem + pool->key_offset,
314 			     (u8 *)new + pool->key_offset, pool->key_size);
315 
316 		if (cmp == 0) {
317 			pr_warn("key already exists!\n");
318 			goto out;
319 		}
320 
321 		if (cmp > 0)
322 			link = &(*link)->rb_left;
323 		else
324 			link = &(*link)->rb_right;
325 	}
326 
327 	rb_link_node(&new->node, parent, link);
328 	rb_insert_color(&new->node, &pool->tree);
329 out:
330 	return;
331 }
332 
333 void rxe_add_key(void *arg, void *key)
334 {
335 	struct rxe_pool_entry *elem = arg;
336 	struct rxe_pool *pool = elem->pool;
337 	unsigned long flags;
338 
339 	write_lock_irqsave(&pool->pool_lock, flags);
340 	memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
341 	insert_key(pool, elem);
342 	write_unlock_irqrestore(&pool->pool_lock, flags);
343 }
344 
345 void rxe_drop_key(void *arg)
346 {
347 	struct rxe_pool_entry *elem = arg;
348 	struct rxe_pool *pool = elem->pool;
349 	unsigned long flags;
350 
351 	write_lock_irqsave(&pool->pool_lock, flags);
352 	rb_erase(&elem->node, &pool->tree);
353 	write_unlock_irqrestore(&pool->pool_lock, flags);
354 }
355 
356 void rxe_add_index(void *arg)
357 {
358 	struct rxe_pool_entry *elem = arg;
359 	struct rxe_pool *pool = elem->pool;
360 	unsigned long flags;
361 
362 	write_lock_irqsave(&pool->pool_lock, flags);
363 	elem->index = alloc_index(pool);
364 	insert_index(pool, elem);
365 	write_unlock_irqrestore(&pool->pool_lock, flags);
366 }
367 
368 void rxe_drop_index(void *arg)
369 {
370 	struct rxe_pool_entry *elem = arg;
371 	struct rxe_pool *pool = elem->pool;
372 	unsigned long flags;
373 
374 	write_lock_irqsave(&pool->pool_lock, flags);
375 	clear_bit(elem->index - pool->min_index, pool->table);
376 	rb_erase(&elem->node, &pool->tree);
377 	write_unlock_irqrestore(&pool->pool_lock, flags);
378 }
379 
380 void *rxe_alloc(struct rxe_pool *pool)
381 {
382 	struct rxe_pool_entry *elem;
383 	unsigned long flags;
384 
385 	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
386 
387 	read_lock_irqsave(&pool->pool_lock, flags);
388 	if (pool->state != RXE_POOL_STATE_VALID) {
389 		read_unlock_irqrestore(&pool->pool_lock, flags);
390 		return NULL;
391 	}
392 	kref_get(&pool->ref_cnt);
393 	read_unlock_irqrestore(&pool->pool_lock, flags);
394 
395 	kref_get(&pool->rxe->ref_cnt);
396 
397 	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
398 		goto out_put_pool;
399 
400 	elem = kmem_cache_zalloc(pool_cache(pool),
401 				 (pool->flags & RXE_POOL_ATOMIC) ?
402 				 GFP_ATOMIC : GFP_KERNEL);
403 	if (!elem)
404 		goto out_put_pool;
405 
406 	elem->pool = pool;
407 	kref_init(&elem->ref_cnt);
408 
409 	return elem;
410 
411 out_put_pool:
412 	atomic_dec(&pool->num_elem);
413 	rxe_dev_put(pool->rxe);
414 	rxe_pool_put(pool);
415 	return NULL;
416 }
417 
418 void rxe_elem_release(struct kref *kref)
419 {
420 	struct rxe_pool_entry *elem =
421 		container_of(kref, struct rxe_pool_entry, ref_cnt);
422 	struct rxe_pool *pool = elem->pool;
423 
424 	if (pool->cleanup)
425 		pool->cleanup(elem);
426 
427 	kmem_cache_free(pool_cache(pool), elem);
428 	atomic_dec(&pool->num_elem);
429 	rxe_dev_put(pool->rxe);
430 	rxe_pool_put(pool);
431 }
432 
433 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
434 {
435 	struct rb_node *node = NULL;
436 	struct rxe_pool_entry *elem = NULL;
437 	unsigned long flags;
438 
439 	read_lock_irqsave(&pool->pool_lock, flags);
440 
441 	if (pool->state != RXE_POOL_STATE_VALID)
442 		goto out;
443 
444 	node = pool->tree.rb_node;
445 
446 	while (node) {
447 		elem = rb_entry(node, struct rxe_pool_entry, node);
448 
449 		if (elem->index > index)
450 			node = node->rb_left;
451 		else if (elem->index < index)
452 			node = node->rb_right;
453 		else {
454 			kref_get(&elem->ref_cnt);
455 			break;
456 		}
457 	}
458 
459 out:
460 	read_unlock_irqrestore(&pool->pool_lock, flags);
461 	return node ? elem : NULL;
462 }
463 
464 void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
465 {
466 	struct rb_node *node = NULL;
467 	struct rxe_pool_entry *elem = NULL;
468 	int cmp;
469 	unsigned long flags;
470 
471 	read_lock_irqsave(&pool->pool_lock, flags);
472 
473 	if (pool->state != RXE_POOL_STATE_VALID)
474 		goto out;
475 
476 	node = pool->tree.rb_node;
477 
478 	while (node) {
479 		elem = rb_entry(node, struct rxe_pool_entry, node);
480 
481 		cmp = memcmp((u8 *)elem + pool->key_offset,
482 			     key, pool->key_size);
483 
484 		if (cmp > 0)
485 			node = node->rb_left;
486 		else if (cmp < 0)
487 			node = node->rb_right;
488 		else
489 			break;
490 	}
491 
492 	if (node)
493 		kref_get(&elem->ref_cnt);
494 
495 out:
496 	read_unlock_irqrestore(&pool->pool_lock, flags);
497 	return node ? elem : NULL;
498 }
499