1 /*
2  * Copyright (c) 2006, 2007 Cisco Systems, Inc.  All rights reserved.
3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/errno.h>
35 #include <linux/slab.h>
36 #include <linux/mm.h>
37 #include <linux/export.h>
38 #include <linux/bitmap.h>
39 #include <linux/dma-mapping.h>
40 #include <linux/vmalloc.h>
41 
42 #include "mlx4.h"
43 
44 u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap)
45 {
46 	u32 obj;
47 
48 	spin_lock(&bitmap->lock);
49 
50 	obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
51 	if (obj >= bitmap->max) {
52 		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
53 				& bitmap->mask;
54 		obj = find_first_zero_bit(bitmap->table, bitmap->max);
55 	}
56 
57 	if (obj < bitmap->max) {
58 		set_bit(obj, bitmap->table);
59 		bitmap->last = (obj + 1);
60 		if (bitmap->last == bitmap->max)
61 			bitmap->last = 0;
62 		obj |= bitmap->top;
63 	} else
64 		obj = -1;
65 
66 	if (obj != -1)
67 		--bitmap->avail;
68 
69 	spin_unlock(&bitmap->lock);
70 
71 	return obj;
72 }
73 
74 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr)
75 {
76 	mlx4_bitmap_free_range(bitmap, obj, 1, use_rr);
77 }
78 
79 u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align)
80 {
81 	u32 obj;
82 
83 	if (likely(cnt == 1 && align == 1))
84 		return mlx4_bitmap_alloc(bitmap);
85 
86 	spin_lock(&bitmap->lock);
87 
88 	obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
89 				bitmap->last, cnt, align - 1);
90 	if (obj >= bitmap->max) {
91 		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
92 				& bitmap->mask;
93 		obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
94 						0, cnt, align - 1);
95 	}
96 
97 	if (obj < bitmap->max) {
98 		bitmap_set(bitmap->table, obj, cnt);
99 		if (obj == bitmap->last) {
100 			bitmap->last = (obj + cnt);
101 			if (bitmap->last >= bitmap->max)
102 				bitmap->last = 0;
103 		}
104 		obj |= bitmap->top;
105 	} else
106 		obj = -1;
107 
108 	if (obj != -1)
109 		bitmap->avail -= cnt;
110 
111 	spin_unlock(&bitmap->lock);
112 
113 	return obj;
114 }
115 
116 u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap)
117 {
118 	return bitmap->avail;
119 }
120 
121 void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt,
122 			    int use_rr)
123 {
124 	obj &= bitmap->max + bitmap->reserved_top - 1;
125 
126 	spin_lock(&bitmap->lock);
127 	if (!use_rr) {
128 		bitmap->last = min(bitmap->last, obj);
129 		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
130 				& bitmap->mask;
131 	}
132 	bitmap_clear(bitmap->table, obj, cnt);
133 	bitmap->avail += cnt;
134 	spin_unlock(&bitmap->lock);
135 }
136 
137 int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,
138 		     u32 reserved_bot, u32 reserved_top)
139 {
140 	/* num must be a power of 2 */
141 	if (num != roundup_pow_of_two(num))
142 		return -EINVAL;
143 
144 	bitmap->last = 0;
145 	bitmap->top  = 0;
146 	bitmap->max  = num - reserved_top;
147 	bitmap->mask = mask;
148 	bitmap->reserved_top = reserved_top;
149 	bitmap->avail = num - reserved_top - reserved_bot;
150 	spin_lock_init(&bitmap->lock);
151 	bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *
152 				sizeof (long), GFP_KERNEL);
153 	if (!bitmap->table)
154 		return -ENOMEM;
155 
156 	bitmap_set(bitmap->table, 0, reserved_bot);
157 
158 	return 0;
159 }
160 
161 void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
162 {
163 	kfree(bitmap->table);
164 }
165 
166 /*
167  * Handling for queue buffers -- we allocate a bunch of memory and
168  * register it in a memory region at HCA virtual address 0.  If the
169  * requested size is > max_direct, we split the allocation into
170  * multiple pages, so we don't require too much contiguous memory.
171  */
172 
173 int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
174 		   struct mlx4_buf *buf, gfp_t gfp)
175 {
176 	dma_addr_t t;
177 
178 	if (size <= max_direct) {
179 		buf->nbufs        = 1;
180 		buf->npages       = 1;
181 		buf->page_shift   = get_order(size) + PAGE_SHIFT;
182 		buf->direct.buf   = dma_alloc_coherent(&dev->pdev->dev,
183 						       size, &t, gfp);
184 		if (!buf->direct.buf)
185 			return -ENOMEM;
186 
187 		buf->direct.map = t;
188 
189 		while (t & ((1 << buf->page_shift) - 1)) {
190 			--buf->page_shift;
191 			buf->npages *= 2;
192 		}
193 
194 		memset(buf->direct.buf, 0, size);
195 	} else {
196 		int i;
197 
198 		buf->direct.buf  = NULL;
199 		buf->nbufs       = (size + PAGE_SIZE - 1) / PAGE_SIZE;
200 		buf->npages      = buf->nbufs;
201 		buf->page_shift  = PAGE_SHIFT;
202 		buf->page_list   = kcalloc(buf->nbufs, sizeof(*buf->page_list),
203 					   gfp);
204 		if (!buf->page_list)
205 			return -ENOMEM;
206 
207 		for (i = 0; i < buf->nbufs; ++i) {
208 			buf->page_list[i].buf =
209 				dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
210 						   &t, gfp);
211 			if (!buf->page_list[i].buf)
212 				goto err_free;
213 
214 			buf->page_list[i].map = t;
215 
216 			memset(buf->page_list[i].buf, 0, PAGE_SIZE);
217 		}
218 
219 		if (BITS_PER_LONG == 64) {
220 			struct page **pages;
221 			pages = kmalloc(sizeof *pages * buf->nbufs, gfp);
222 			if (!pages)
223 				goto err_free;
224 			for (i = 0; i < buf->nbufs; ++i)
225 				pages[i] = virt_to_page(buf->page_list[i].buf);
226 			buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
227 			kfree(pages);
228 			if (!buf->direct.buf)
229 				goto err_free;
230 		}
231 	}
232 
233 	return 0;
234 
235 err_free:
236 	mlx4_buf_free(dev, size, buf);
237 
238 	return -ENOMEM;
239 }
240 EXPORT_SYMBOL_GPL(mlx4_buf_alloc);
241 
242 void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf)
243 {
244 	int i;
245 
246 	if (buf->nbufs == 1)
247 		dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf,
248 				  buf->direct.map);
249 	else {
250 		if (BITS_PER_LONG == 64 && buf->direct.buf)
251 			vunmap(buf->direct.buf);
252 
253 		for (i = 0; i < buf->nbufs; ++i)
254 			if (buf->page_list[i].buf)
255 				dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
256 						  buf->page_list[i].buf,
257 						  buf->page_list[i].map);
258 		kfree(buf->page_list);
259 	}
260 }
261 EXPORT_SYMBOL_GPL(mlx4_buf_free);
262 
263 static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device,
264 						 gfp_t gfp)
265 {
266 	struct mlx4_db_pgdir *pgdir;
267 
268 	pgdir = kzalloc(sizeof *pgdir, gfp);
269 	if (!pgdir)
270 		return NULL;
271 
272 	bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2);
273 	pgdir->bits[0] = pgdir->order0;
274 	pgdir->bits[1] = pgdir->order1;
275 	pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE,
276 					    &pgdir->db_dma, gfp);
277 	if (!pgdir->db_page) {
278 		kfree(pgdir);
279 		return NULL;
280 	}
281 
282 	return pgdir;
283 }
284 
285 static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir,
286 				    struct mlx4_db *db, int order)
287 {
288 	int o;
289 	int i;
290 
291 	for (o = order; o <= 1; ++o) {
292 		i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o);
293 		if (i < MLX4_DB_PER_PAGE >> o)
294 			goto found;
295 	}
296 
297 	return -ENOMEM;
298 
299 found:
300 	clear_bit(i, pgdir->bits[o]);
301 
302 	i <<= o;
303 
304 	if (o > order)
305 		set_bit(i ^ 1, pgdir->bits[order]);
306 
307 	db->u.pgdir = pgdir;
308 	db->index   = i;
309 	db->db      = pgdir->db_page + db->index;
310 	db->dma     = pgdir->db_dma  + db->index * 4;
311 	db->order   = order;
312 
313 	return 0;
314 }
315 
316 int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, gfp_t gfp)
317 {
318 	struct mlx4_priv *priv = mlx4_priv(dev);
319 	struct mlx4_db_pgdir *pgdir;
320 	int ret = 0;
321 
322 	mutex_lock(&priv->pgdir_mutex);
323 
324 	list_for_each_entry(pgdir, &priv->pgdir_list, list)
325 		if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))
326 			goto out;
327 
328 	pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev), gfp);
329 	if (!pgdir) {
330 		ret = -ENOMEM;
331 		goto out;
332 	}
333 
334 	list_add(&pgdir->list, &priv->pgdir_list);
335 
336 	/* This should never fail -- we just allocated an empty page: */
337 	WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order));
338 
339 out:
340 	mutex_unlock(&priv->pgdir_mutex);
341 
342 	return ret;
343 }
344 EXPORT_SYMBOL_GPL(mlx4_db_alloc);
345 
346 void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db)
347 {
348 	struct mlx4_priv *priv = mlx4_priv(dev);
349 	int o;
350 	int i;
351 
352 	mutex_lock(&priv->pgdir_mutex);
353 
354 	o = db->order;
355 	i = db->index;
356 
357 	if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
358 		clear_bit(i ^ 1, db->u.pgdir->order0);
359 		++o;
360 	}
361 	i >>= o;
362 	set_bit(i, db->u.pgdir->bits[o]);
363 
364 	if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) {
365 		dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,
366 				  db->u.pgdir->db_page, db->u.pgdir->db_dma);
367 		list_del(&db->u.pgdir->list);
368 		kfree(db->u.pgdir);
369 	}
370 
371 	mutex_unlock(&priv->pgdir_mutex);
372 }
373 EXPORT_SYMBOL_GPL(mlx4_db_free);
374 
375 int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
376 		       int size, int max_direct)
377 {
378 	int err;
379 
380 	err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL);
381 	if (err)
382 		return err;
383 
384 	*wqres->db.db = 0;
385 
386 	err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL);
387 	if (err)
388 		goto err_db;
389 
390 	err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift,
391 			    &wqres->mtt);
392 	if (err)
393 		goto err_buf;
394 
395 	err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL);
396 	if (err)
397 		goto err_mtt;
398 
399 	return 0;
400 
401 err_mtt:
402 	mlx4_mtt_cleanup(dev, &wqres->mtt);
403 err_buf:
404 	mlx4_buf_free(dev, size, &wqres->buf);
405 err_db:
406 	mlx4_db_free(dev, &wqres->db);
407 
408 	return err;
409 }
410 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res);
411 
412 void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
413 		       int size)
414 {
415 	mlx4_mtt_cleanup(dev, &wqres->mtt);
416 	mlx4_buf_free(dev, size, &wqres->buf);
417 	mlx4_db_free(dev, &wqres->db);
418 }
419 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res);
420