1 /* 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/errno.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/export.h> 38 #include <linux/bitmap.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/vmalloc.h> 41 42 #include "mlx4.h" 43 44 u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) 45 { 46 u32 obj; 47 48 spin_lock(&bitmap->lock); 49 50 obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 51 if (obj >= bitmap->max) { 52 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 53 & bitmap->mask; 54 obj = find_first_zero_bit(bitmap->table, bitmap->max); 55 } 56 57 if (obj < bitmap->max) { 58 set_bit(obj, bitmap->table); 59 bitmap->last = (obj + 1); 60 if (bitmap->last == bitmap->max) 61 bitmap->last = 0; 62 obj |= bitmap->top; 63 } else 64 obj = -1; 65 66 if (obj != -1) 67 --bitmap->avail; 68 69 spin_unlock(&bitmap->lock); 70 71 return obj; 72 } 73 74 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) 75 { 76 mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); 77 } 78 79 static unsigned long find_aligned_range(unsigned long *bitmap, 80 u32 start, u32 nbits, 81 int len, int align, u32 skip_mask) 82 { 83 unsigned long end, i; 84 85 again: 86 start = ALIGN(start, align); 87 88 while ((start < nbits) && (test_bit(start, bitmap) || 89 (start & skip_mask))) 90 start += align; 91 92 if (start >= nbits) 93 return -1; 94 95 end = start+len; 96 if (end > nbits) 97 return -1; 98 99 for (i = start + 1; i < end; i++) { 100 if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { 101 start = i + 1; 102 goto again; 103 } 104 } 105 106 return start; 107 } 108 109 u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, 110 int align, u32 skip_mask) 111 { 112 u32 obj; 113 114 if (likely(cnt == 1 && align == 1 && !skip_mask)) 115 return mlx4_bitmap_alloc(bitmap); 116 117 spin_lock(&bitmap->lock); 118 119 obj = find_aligned_range(bitmap->table, bitmap->last, 120 bitmap->max, cnt, align, skip_mask); 121 if (obj >= bitmap->max) { 122 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 123 & bitmap->mask; 124 obj = find_aligned_range(bitmap->table, 0, bitmap->max, 125 cnt, align, skip_mask); 126 } 127 128 if (obj < bitmap->max) { 129 bitmap_set(bitmap->table, obj, cnt); 130 if (obj == bitmap->last) { 131 bitmap->last = (obj + cnt); 132 if (bitmap->last >= bitmap->max) 133 bitmap->last = 0; 134 } 135 obj |= bitmap->top; 136 } else 137 obj = -1; 138 139 if (obj != -1) 140 bitmap->avail -= cnt; 141 142 spin_unlock(&bitmap->lock); 143 144 return obj; 145 } 146 147 u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) 148 { 149 return bitmap->avail; 150 } 151 152 static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj) 153 { 154 return obj & (bitmap->max + bitmap->reserved_top - 1); 155 } 156 157 void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, 158 int use_rr) 159 { 160 obj &= bitmap->max + bitmap->reserved_top - 1; 161 162 spin_lock(&bitmap->lock); 163 if (!use_rr) { 164 bitmap->last = min(bitmap->last, obj); 165 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 166 & bitmap->mask; 167 } 168 bitmap_clear(bitmap->table, obj, cnt); 169 bitmap->avail += cnt; 170 spin_unlock(&bitmap->lock); 171 } 172 173 int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, 174 u32 reserved_bot, u32 reserved_top) 175 { 176 /* num must be a power of 2 */ 177 if (num != roundup_pow_of_two(num)) 178 return -EINVAL; 179 180 bitmap->last = 0; 181 bitmap->top = 0; 182 bitmap->max = num - reserved_top; 183 bitmap->mask = mask; 184 bitmap->reserved_top = reserved_top; 185 bitmap->avail = num - reserved_top - reserved_bot; 186 bitmap->effective_len = bitmap->avail; 187 spin_lock_init(&bitmap->lock); 188 bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * 189 sizeof (long), GFP_KERNEL); 190 if (!bitmap->table) 191 return -ENOMEM; 192 193 bitmap_set(bitmap->table, 0, reserved_bot); 194 195 return 0; 196 } 197 198 void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) 199 { 200 kfree(bitmap->table); 201 } 202 203 struct mlx4_zone_allocator { 204 struct list_head entries; 205 struct list_head prios; 206 u32 last_uid; 207 u32 mask; 208 /* protect the zone_allocator from concurrent accesses */ 209 spinlock_t lock; 210 enum mlx4_zone_alloc_flags flags; 211 }; 212 213 struct mlx4_zone_entry { 214 struct list_head list; 215 struct list_head prio_list; 216 u32 uid; 217 struct mlx4_zone_allocator *allocator; 218 struct mlx4_bitmap *bitmap; 219 int use_rr; 220 int priority; 221 int offset; 222 enum mlx4_zone_flags flags; 223 }; 224 225 struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags) 226 { 227 struct mlx4_zone_allocator *zones = kmalloc(sizeof(*zones), GFP_KERNEL); 228 229 if (NULL == zones) 230 return NULL; 231 232 INIT_LIST_HEAD(&zones->entries); 233 INIT_LIST_HEAD(&zones->prios); 234 spin_lock_init(&zones->lock); 235 zones->last_uid = 0; 236 zones->mask = 0; 237 zones->flags = flags; 238 239 return zones; 240 } 241 242 int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc, 243 struct mlx4_bitmap *bitmap, 244 u32 flags, 245 int priority, 246 int offset, 247 u32 *puid) 248 { 249 u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1); 250 struct mlx4_zone_entry *it; 251 struct mlx4_zone_entry *zone = kmalloc(sizeof(*zone), GFP_KERNEL); 252 253 if (NULL == zone) 254 return -ENOMEM; 255 256 zone->flags = flags; 257 zone->bitmap = bitmap; 258 zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0; 259 zone->priority = priority; 260 zone->offset = offset; 261 262 spin_lock(&zone_alloc->lock); 263 264 zone->uid = zone_alloc->last_uid++; 265 zone->allocator = zone_alloc; 266 267 if (zone_alloc->mask < mask) 268 zone_alloc->mask = mask; 269 270 list_for_each_entry(it, &zone_alloc->prios, prio_list) 271 if (it->priority >= priority) 272 break; 273 274 if (&it->prio_list == &zone_alloc->prios || it->priority > priority) 275 list_add_tail(&zone->prio_list, &it->prio_list); 276 list_add_tail(&zone->list, &it->list); 277 278 spin_unlock(&zone_alloc->lock); 279 280 *puid = zone->uid; 281 282 return 0; 283 } 284 285 /* Should be called under a lock */ 286 static int __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry) 287 { 288 struct mlx4_zone_allocator *zone_alloc = entry->allocator; 289 290 if (!list_empty(&entry->prio_list)) { 291 /* Check if we need to add an alternative node to the prio list */ 292 if (!list_is_last(&entry->list, &zone_alloc->entries)) { 293 struct mlx4_zone_entry *next = list_first_entry(&entry->list, 294 typeof(*next), 295 list); 296 297 if (next->priority == entry->priority) 298 list_add_tail(&next->prio_list, &entry->prio_list); 299 } 300 301 list_del(&entry->prio_list); 302 } 303 304 list_del(&entry->list); 305 306 if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) { 307 u32 mask = 0; 308 struct mlx4_zone_entry *it; 309 310 list_for_each_entry(it, &zone_alloc->prios, prio_list) { 311 u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1); 312 313 if (mask < cur_mask) 314 mask = cur_mask; 315 } 316 zone_alloc->mask = mask; 317 } 318 319 return 0; 320 } 321 322 void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc) 323 { 324 struct mlx4_zone_entry *zone, *tmp; 325 326 spin_lock(&zone_alloc->lock); 327 328 list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) { 329 list_del(&zone->list); 330 list_del(&zone->prio_list); 331 kfree(zone); 332 } 333 334 spin_unlock(&zone_alloc->lock); 335 kfree(zone_alloc); 336 } 337 338 /* Should be called under a lock */ 339 static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count, 340 int align, u32 skip_mask, u32 *puid) 341 { 342 u32 uid; 343 u32 res; 344 struct mlx4_zone_allocator *zone_alloc = zone->allocator; 345 struct mlx4_zone_entry *curr_node; 346 347 res = mlx4_bitmap_alloc_range(zone->bitmap, count, 348 align, skip_mask); 349 350 if (res != (u32)-1) { 351 res += zone->offset; 352 uid = zone->uid; 353 goto out; 354 } 355 356 list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) { 357 if (unlikely(curr_node->priority == zone->priority)) 358 break; 359 } 360 361 if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) { 362 struct mlx4_zone_entry *it = curr_node; 363 364 list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) { 365 res = mlx4_bitmap_alloc_range(it->bitmap, count, 366 align, skip_mask); 367 if (res != (u32)-1) { 368 res += it->offset; 369 uid = it->uid; 370 goto out; 371 } 372 } 373 } 374 375 if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) { 376 struct mlx4_zone_entry *it = curr_node; 377 378 list_for_each_entry_from(it, &zone_alloc->entries, list) { 379 if (unlikely(it == zone)) 380 continue; 381 382 if (unlikely(it->priority != curr_node->priority)) 383 break; 384 385 res = mlx4_bitmap_alloc_range(it->bitmap, count, 386 align, skip_mask); 387 if (res != (u32)-1) { 388 res += it->offset; 389 uid = it->uid; 390 goto out; 391 } 392 } 393 } 394 395 if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) { 396 if (list_is_last(&curr_node->prio_list, &zone_alloc->prios)) 397 goto out; 398 399 curr_node = list_first_entry(&curr_node->prio_list, 400 typeof(*curr_node), 401 prio_list); 402 403 list_for_each_entry_from(curr_node, &zone_alloc->entries, list) { 404 res = mlx4_bitmap_alloc_range(curr_node->bitmap, count, 405 align, skip_mask); 406 if (res != (u32)-1) { 407 res += curr_node->offset; 408 uid = curr_node->uid; 409 goto out; 410 } 411 } 412 } 413 414 out: 415 if (NULL != puid && res != (u32)-1) 416 *puid = uid; 417 return res; 418 } 419 420 /* Should be called under a lock */ 421 static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj, 422 u32 count) 423 { 424 mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr); 425 } 426 427 /* Should be called under a lock */ 428 static struct mlx4_zone_entry *__mlx4_find_zone_by_uid( 429 struct mlx4_zone_allocator *zones, u32 uid) 430 { 431 struct mlx4_zone_entry *zone; 432 433 list_for_each_entry(zone, &zones->entries, list) { 434 if (zone->uid == uid) 435 return zone; 436 } 437 438 return NULL; 439 } 440 441 struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid) 442 { 443 struct mlx4_zone_entry *zone; 444 struct mlx4_bitmap *bitmap; 445 446 spin_lock(&zones->lock); 447 448 zone = __mlx4_find_zone_by_uid(zones, uid); 449 450 bitmap = zone == NULL ? NULL : zone->bitmap; 451 452 spin_unlock(&zones->lock); 453 454 return bitmap; 455 } 456 457 int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid) 458 { 459 struct mlx4_zone_entry *zone; 460 int res; 461 462 spin_lock(&zones->lock); 463 464 zone = __mlx4_find_zone_by_uid(zones, uid); 465 466 if (NULL == zone) { 467 res = -1; 468 goto out; 469 } 470 471 res = __mlx4_zone_remove_one_entry(zone); 472 473 out: 474 spin_unlock(&zones->lock); 475 kfree(zone); 476 477 return res; 478 } 479 480 /* Should be called under a lock */ 481 static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique( 482 struct mlx4_zone_allocator *zones, u32 obj) 483 { 484 struct mlx4_zone_entry *zone, *zone_candidate = NULL; 485 u32 dist = (u32)-1; 486 487 /* Search for the smallest zone that this obj could be 488 * allocated from. This is done in order to handle 489 * situations when small bitmaps are allocated from bigger 490 * bitmaps (and the allocated space is marked as reserved in 491 * the bigger bitmap. 492 */ 493 list_for_each_entry(zone, &zones->entries, list) { 494 if (obj >= zone->offset) { 495 u32 mobj = (obj - zone->offset) & zones->mask; 496 497 if (mobj < zone->bitmap->max) { 498 u32 curr_dist = zone->bitmap->effective_len; 499 500 if (curr_dist < dist) { 501 dist = curr_dist; 502 zone_candidate = zone; 503 } 504 } 505 } 506 } 507 508 return zone_candidate; 509 } 510 511 u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count, 512 int align, u32 skip_mask, u32 *puid) 513 { 514 struct mlx4_zone_entry *zone; 515 int res = -1; 516 517 spin_lock(&zones->lock); 518 519 zone = __mlx4_find_zone_by_uid(zones, uid); 520 521 if (NULL == zone) 522 goto out; 523 524 res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid); 525 526 out: 527 spin_unlock(&zones->lock); 528 529 return res; 530 } 531 532 u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count) 533 { 534 struct mlx4_zone_entry *zone; 535 int res = 0; 536 537 spin_lock(&zones->lock); 538 539 zone = __mlx4_find_zone_by_uid(zones, uid); 540 541 if (NULL == zone) { 542 res = -1; 543 goto out; 544 } 545 546 __mlx4_free_from_zone(zone, obj, count); 547 548 out: 549 spin_unlock(&zones->lock); 550 551 return res; 552 } 553 554 u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count) 555 { 556 struct mlx4_zone_entry *zone; 557 int res; 558 559 if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP)) 560 return -EFAULT; 561 562 spin_lock(&zones->lock); 563 564 zone = __mlx4_find_zone_by_uid_unique(zones, obj); 565 566 if (NULL == zone) { 567 res = -1; 568 goto out; 569 } 570 571 __mlx4_free_from_zone(zone, obj, count); 572 res = 0; 573 574 out: 575 spin_unlock(&zones->lock); 576 577 return res; 578 } 579 /* 580 * Handling for queue buffers -- we allocate a bunch of memory and 581 * register it in a memory region at HCA virtual address 0. If the 582 * requested size is > max_direct, we split the allocation into 583 * multiple pages, so we don't require too much contiguous memory. 584 */ 585 586 int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 587 struct mlx4_buf *buf, gfp_t gfp) 588 { 589 dma_addr_t t; 590 591 if (size <= max_direct) { 592 buf->nbufs = 1; 593 buf->npages = 1; 594 buf->page_shift = get_order(size) + PAGE_SHIFT; 595 buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 596 size, &t, gfp); 597 if (!buf->direct.buf) 598 return -ENOMEM; 599 600 buf->direct.map = t; 601 602 while (t & ((1 << buf->page_shift) - 1)) { 603 --buf->page_shift; 604 buf->npages *= 2; 605 } 606 607 memset(buf->direct.buf, 0, size); 608 } else { 609 int i; 610 611 buf->direct.buf = NULL; 612 buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 613 buf->npages = buf->nbufs; 614 buf->page_shift = PAGE_SHIFT; 615 buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 616 gfp); 617 if (!buf->page_list) 618 return -ENOMEM; 619 620 for (i = 0; i < buf->nbufs; ++i) { 621 buf->page_list[i].buf = 622 dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 623 &t, gfp); 624 if (!buf->page_list[i].buf) 625 goto err_free; 626 627 buf->page_list[i].map = t; 628 629 memset(buf->page_list[i].buf, 0, PAGE_SIZE); 630 } 631 632 if (BITS_PER_LONG == 64) { 633 struct page **pages; 634 pages = kmalloc(sizeof *pages * buf->nbufs, gfp); 635 if (!pages) 636 goto err_free; 637 for (i = 0; i < buf->nbufs; ++i) 638 pages[i] = virt_to_page(buf->page_list[i].buf); 639 buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); 640 kfree(pages); 641 if (!buf->direct.buf) 642 goto err_free; 643 } 644 } 645 646 return 0; 647 648 err_free: 649 mlx4_buf_free(dev, size, buf); 650 651 return -ENOMEM; 652 } 653 EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 654 655 void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) 656 { 657 int i; 658 659 if (buf->nbufs == 1) 660 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 661 buf->direct.map); 662 else { 663 if (BITS_PER_LONG == 64 && buf->direct.buf) 664 vunmap(buf->direct.buf); 665 666 for (i = 0; i < buf->nbufs; ++i) 667 if (buf->page_list[i].buf) 668 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 669 buf->page_list[i].buf, 670 buf->page_list[i].map); 671 kfree(buf->page_list); 672 } 673 } 674 EXPORT_SYMBOL_GPL(mlx4_buf_free); 675 676 static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device, 677 gfp_t gfp) 678 { 679 struct mlx4_db_pgdir *pgdir; 680 681 pgdir = kzalloc(sizeof *pgdir, gfp); 682 if (!pgdir) 683 return NULL; 684 685 bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 686 pgdir->bits[0] = pgdir->order0; 687 pgdir->bits[1] = pgdir->order1; 688 pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, 689 &pgdir->db_dma, gfp); 690 if (!pgdir->db_page) { 691 kfree(pgdir); 692 return NULL; 693 } 694 695 return pgdir; 696 } 697 698 static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 699 struct mlx4_db *db, int order) 700 { 701 int o; 702 int i; 703 704 for (o = order; o <= 1; ++o) { 705 i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 706 if (i < MLX4_DB_PER_PAGE >> o) 707 goto found; 708 } 709 710 return -ENOMEM; 711 712 found: 713 clear_bit(i, pgdir->bits[o]); 714 715 i <<= o; 716 717 if (o > order) 718 set_bit(i ^ 1, pgdir->bits[order]); 719 720 db->u.pgdir = pgdir; 721 db->index = i; 722 db->db = pgdir->db_page + db->index; 723 db->dma = pgdir->db_dma + db->index * 4; 724 db->order = order; 725 726 return 0; 727 } 728 729 int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, gfp_t gfp) 730 { 731 struct mlx4_priv *priv = mlx4_priv(dev); 732 struct mlx4_db_pgdir *pgdir; 733 int ret = 0; 734 735 mutex_lock(&priv->pgdir_mutex); 736 737 list_for_each_entry(pgdir, &priv->pgdir_list, list) 738 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 739 goto out; 740 741 pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev), gfp); 742 if (!pgdir) { 743 ret = -ENOMEM; 744 goto out; 745 } 746 747 list_add(&pgdir->list, &priv->pgdir_list); 748 749 /* This should never fail -- we just allocated an empty page: */ 750 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); 751 752 out: 753 mutex_unlock(&priv->pgdir_mutex); 754 755 return ret; 756 } 757 EXPORT_SYMBOL_GPL(mlx4_db_alloc); 758 759 void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) 760 { 761 struct mlx4_priv *priv = mlx4_priv(dev); 762 int o; 763 int i; 764 765 mutex_lock(&priv->pgdir_mutex); 766 767 o = db->order; 768 i = db->index; 769 770 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 771 clear_bit(i ^ 1, db->u.pgdir->order0); 772 ++o; 773 } 774 i >>= o; 775 set_bit(i, db->u.pgdir->bits[o]); 776 777 if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 778 dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, 779 db->u.pgdir->db_page, db->u.pgdir->db_dma); 780 list_del(&db->u.pgdir->list); 781 kfree(db->u.pgdir); 782 } 783 784 mutex_unlock(&priv->pgdir_mutex); 785 } 786 EXPORT_SYMBOL_GPL(mlx4_db_free); 787 788 int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 789 int size, int max_direct) 790 { 791 int err; 792 793 err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL); 794 if (err) 795 return err; 796 797 *wqres->db.db = 0; 798 799 err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL); 800 if (err) 801 goto err_db; 802 803 err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 804 &wqres->mtt); 805 if (err) 806 goto err_buf; 807 808 err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL); 809 if (err) 810 goto err_mtt; 811 812 return 0; 813 814 err_mtt: 815 mlx4_mtt_cleanup(dev, &wqres->mtt); 816 err_buf: 817 mlx4_buf_free(dev, size, &wqres->buf); 818 err_db: 819 mlx4_db_free(dev, &wqres->db); 820 821 return err; 822 } 823 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 824 825 void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 826 int size) 827 { 828 mlx4_mtt_cleanup(dev, &wqres->mtt); 829 mlx4_buf_free(dev, size, &wqres->buf); 830 mlx4_db_free(dev, &wqres->db); 831 } 832 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); 833