1 /* 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/errno.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/export.h> 38 #include <linux/bitmap.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/vmalloc.h> 41 42 #include "mlx4.h" 43 44 u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) 45 { 46 u32 obj; 47 48 spin_lock(&bitmap->lock); 49 50 obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 51 if (obj >= bitmap->max) { 52 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 53 & bitmap->mask; 54 obj = find_first_zero_bit(bitmap->table, bitmap->max); 55 } 56 57 if (obj < bitmap->max) { 58 set_bit(obj, bitmap->table); 59 bitmap->last = (obj + 1); 60 if (bitmap->last == bitmap->max) 61 bitmap->last = 0; 62 obj |= bitmap->top; 63 } else 64 obj = -1; 65 66 if (obj != -1) 67 --bitmap->avail; 68 69 spin_unlock(&bitmap->lock); 70 71 return obj; 72 } 73 74 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) 75 { 76 mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); 77 } 78 79 static unsigned long find_aligned_range(unsigned long *bitmap, 80 u32 start, u32 nbits, 81 int len, int align, u32 skip_mask) 82 { 83 unsigned long end, i; 84 85 again: 86 start = ALIGN(start, align); 87 88 while ((start < nbits) && (test_bit(start, bitmap) || 89 (start & skip_mask))) 90 start += align; 91 92 if (start >= nbits) 93 return -1; 94 95 end = start+len; 96 if (end > nbits) 97 return -1; 98 99 for (i = start + 1; i < end; i++) { 100 if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { 101 start = i + 1; 102 goto again; 103 } 104 } 105 106 return start; 107 } 108 109 u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, 110 int align, u32 skip_mask) 111 { 112 u32 obj; 113 114 if (likely(cnt == 1 && align == 1 && !skip_mask)) 115 return mlx4_bitmap_alloc(bitmap); 116 117 spin_lock(&bitmap->lock); 118 119 obj = find_aligned_range(bitmap->table, bitmap->last, 120 bitmap->max, cnt, align, skip_mask); 121 if (obj >= bitmap->max) { 122 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 123 & bitmap->mask; 124 obj = find_aligned_range(bitmap->table, 0, bitmap->max, 125 cnt, align, skip_mask); 126 } 127 128 if (obj < bitmap->max) { 129 bitmap_set(bitmap->table, obj, cnt); 130 if (obj == bitmap->last) { 131 bitmap->last = (obj + cnt); 132 if (bitmap->last >= bitmap->max) 133 bitmap->last = 0; 134 } 135 obj |= bitmap->top; 136 } else 137 obj = -1; 138 139 if (obj != -1) 140 bitmap->avail -= cnt; 141 142 spin_unlock(&bitmap->lock); 143 144 return obj; 145 } 146 147 u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) 148 { 149 return bitmap->avail; 150 } 151 152 void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, 153 int use_rr) 154 { 155 obj &= bitmap->max + bitmap->reserved_top - 1; 156 157 spin_lock(&bitmap->lock); 158 if (!use_rr) { 159 bitmap->last = min(bitmap->last, obj); 160 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 161 & bitmap->mask; 162 } 163 bitmap_clear(bitmap->table, obj, cnt); 164 bitmap->avail += cnt; 165 spin_unlock(&bitmap->lock); 166 } 167 168 int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, 169 u32 reserved_bot, u32 reserved_top) 170 { 171 /* num must be a power of 2 */ 172 if (num != roundup_pow_of_two(num)) 173 return -EINVAL; 174 175 bitmap->last = 0; 176 bitmap->top = 0; 177 bitmap->max = num - reserved_top; 178 bitmap->mask = mask; 179 bitmap->reserved_top = reserved_top; 180 bitmap->avail = num - reserved_top - reserved_bot; 181 spin_lock_init(&bitmap->lock); 182 bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * 183 sizeof (long), GFP_KERNEL); 184 if (!bitmap->table) 185 return -ENOMEM; 186 187 bitmap_set(bitmap->table, 0, reserved_bot); 188 189 return 0; 190 } 191 192 void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) 193 { 194 kfree(bitmap->table); 195 } 196 197 /* 198 * Handling for queue buffers -- we allocate a bunch of memory and 199 * register it in a memory region at HCA virtual address 0. If the 200 * requested size is > max_direct, we split the allocation into 201 * multiple pages, so we don't require too much contiguous memory. 202 */ 203 204 int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 205 struct mlx4_buf *buf, gfp_t gfp) 206 { 207 dma_addr_t t; 208 209 if (size <= max_direct) { 210 buf->nbufs = 1; 211 buf->npages = 1; 212 buf->page_shift = get_order(size) + PAGE_SHIFT; 213 buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 214 size, &t, gfp); 215 if (!buf->direct.buf) 216 return -ENOMEM; 217 218 buf->direct.map = t; 219 220 while (t & ((1 << buf->page_shift) - 1)) { 221 --buf->page_shift; 222 buf->npages *= 2; 223 } 224 225 memset(buf->direct.buf, 0, size); 226 } else { 227 int i; 228 229 buf->direct.buf = NULL; 230 buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 231 buf->npages = buf->nbufs; 232 buf->page_shift = PAGE_SHIFT; 233 buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 234 gfp); 235 if (!buf->page_list) 236 return -ENOMEM; 237 238 for (i = 0; i < buf->nbufs; ++i) { 239 buf->page_list[i].buf = 240 dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 241 &t, gfp); 242 if (!buf->page_list[i].buf) 243 goto err_free; 244 245 buf->page_list[i].map = t; 246 247 memset(buf->page_list[i].buf, 0, PAGE_SIZE); 248 } 249 250 if (BITS_PER_LONG == 64) { 251 struct page **pages; 252 pages = kmalloc(sizeof *pages * buf->nbufs, gfp); 253 if (!pages) 254 goto err_free; 255 for (i = 0; i < buf->nbufs; ++i) 256 pages[i] = virt_to_page(buf->page_list[i].buf); 257 buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); 258 kfree(pages); 259 if (!buf->direct.buf) 260 goto err_free; 261 } 262 } 263 264 return 0; 265 266 err_free: 267 mlx4_buf_free(dev, size, buf); 268 269 return -ENOMEM; 270 } 271 EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 272 273 void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) 274 { 275 int i; 276 277 if (buf->nbufs == 1) 278 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 279 buf->direct.map); 280 else { 281 if (BITS_PER_LONG == 64 && buf->direct.buf) 282 vunmap(buf->direct.buf); 283 284 for (i = 0; i < buf->nbufs; ++i) 285 if (buf->page_list[i].buf) 286 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 287 buf->page_list[i].buf, 288 buf->page_list[i].map); 289 kfree(buf->page_list); 290 } 291 } 292 EXPORT_SYMBOL_GPL(mlx4_buf_free); 293 294 static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device, 295 gfp_t gfp) 296 { 297 struct mlx4_db_pgdir *pgdir; 298 299 pgdir = kzalloc(sizeof *pgdir, gfp); 300 if (!pgdir) 301 return NULL; 302 303 bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 304 pgdir->bits[0] = pgdir->order0; 305 pgdir->bits[1] = pgdir->order1; 306 pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, 307 &pgdir->db_dma, gfp); 308 if (!pgdir->db_page) { 309 kfree(pgdir); 310 return NULL; 311 } 312 313 return pgdir; 314 } 315 316 static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 317 struct mlx4_db *db, int order) 318 { 319 int o; 320 int i; 321 322 for (o = order; o <= 1; ++o) { 323 i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 324 if (i < MLX4_DB_PER_PAGE >> o) 325 goto found; 326 } 327 328 return -ENOMEM; 329 330 found: 331 clear_bit(i, pgdir->bits[o]); 332 333 i <<= o; 334 335 if (o > order) 336 set_bit(i ^ 1, pgdir->bits[order]); 337 338 db->u.pgdir = pgdir; 339 db->index = i; 340 db->db = pgdir->db_page + db->index; 341 db->dma = pgdir->db_dma + db->index * 4; 342 db->order = order; 343 344 return 0; 345 } 346 347 int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, gfp_t gfp) 348 { 349 struct mlx4_priv *priv = mlx4_priv(dev); 350 struct mlx4_db_pgdir *pgdir; 351 int ret = 0; 352 353 mutex_lock(&priv->pgdir_mutex); 354 355 list_for_each_entry(pgdir, &priv->pgdir_list, list) 356 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 357 goto out; 358 359 pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev), gfp); 360 if (!pgdir) { 361 ret = -ENOMEM; 362 goto out; 363 } 364 365 list_add(&pgdir->list, &priv->pgdir_list); 366 367 /* This should never fail -- we just allocated an empty page: */ 368 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); 369 370 out: 371 mutex_unlock(&priv->pgdir_mutex); 372 373 return ret; 374 } 375 EXPORT_SYMBOL_GPL(mlx4_db_alloc); 376 377 void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) 378 { 379 struct mlx4_priv *priv = mlx4_priv(dev); 380 int o; 381 int i; 382 383 mutex_lock(&priv->pgdir_mutex); 384 385 o = db->order; 386 i = db->index; 387 388 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 389 clear_bit(i ^ 1, db->u.pgdir->order0); 390 ++o; 391 } 392 i >>= o; 393 set_bit(i, db->u.pgdir->bits[o]); 394 395 if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 396 dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, 397 db->u.pgdir->db_page, db->u.pgdir->db_dma); 398 list_del(&db->u.pgdir->list); 399 kfree(db->u.pgdir); 400 } 401 402 mutex_unlock(&priv->pgdir_mutex); 403 } 404 EXPORT_SYMBOL_GPL(mlx4_db_free); 405 406 int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 407 int size, int max_direct) 408 { 409 int err; 410 411 err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL); 412 if (err) 413 return err; 414 415 *wqres->db.db = 0; 416 417 err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL); 418 if (err) 419 goto err_db; 420 421 err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 422 &wqres->mtt); 423 if (err) 424 goto err_buf; 425 426 err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL); 427 if (err) 428 goto err_mtt; 429 430 return 0; 431 432 err_mtt: 433 mlx4_mtt_cleanup(dev, &wqres->mtt); 434 err_buf: 435 mlx4_buf_free(dev, size, &wqres->buf); 436 err_db: 437 mlx4_db_free(dev, &wqres->db); 438 439 return err; 440 } 441 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 442 443 void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 444 int size) 445 { 446 mlx4_mtt_cleanup(dev, &wqres->mtt); 447 mlx4_buf_free(dev, size, &wqres->buf); 448 mlx4_db_free(dev, &wqres->db); 449 } 450 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); 451