1 /* 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/errno.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/export.h> 38 #include <linux/bitmap.h> 39 #include <linux/dma-mapping.h> 40 #include <linux/vmalloc.h> 41 42 #include "mlx4.h" 43 44 u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) 45 { 46 u32 obj; 47 48 spin_lock(&bitmap->lock); 49 50 obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 51 if (obj >= bitmap->max) { 52 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 53 & bitmap->mask; 54 obj = find_first_zero_bit(bitmap->table, bitmap->max); 55 } 56 57 if (obj < bitmap->max) { 58 set_bit(obj, bitmap->table); 59 bitmap->last = (obj + 1); 60 if (bitmap->last == bitmap->max) 61 bitmap->last = 0; 62 obj |= bitmap->top; 63 } else 64 obj = -1; 65 66 if (obj != -1) 67 --bitmap->avail; 68 69 spin_unlock(&bitmap->lock); 70 71 return obj; 72 } 73 74 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) 75 { 76 mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); 77 } 78 79 static unsigned long find_aligned_range(unsigned long *bitmap, 80 u32 start, u32 nbits, 81 int len, int align, u32 skip_mask) 82 { 83 unsigned long end, i; 84 85 again: 86 start = ALIGN(start, align); 87 88 while ((start < nbits) && (test_bit(start, bitmap) || 89 (start & skip_mask))) 90 start += align; 91 92 if (start >= nbits) 93 return -1; 94 95 end = start+len; 96 if (end > nbits) 97 return -1; 98 99 for (i = start + 1; i < end; i++) { 100 if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { 101 start = i + 1; 102 goto again; 103 } 104 } 105 106 return start; 107 } 108 109 u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, 110 int align, u32 skip_mask) 111 { 112 u32 obj; 113 114 if (likely(cnt == 1 && align == 1 && !skip_mask)) 115 return mlx4_bitmap_alloc(bitmap); 116 117 spin_lock(&bitmap->lock); 118 119 obj = find_aligned_range(bitmap->table, bitmap->last, 120 bitmap->max, cnt, align, skip_mask); 121 if (obj >= bitmap->max) { 122 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 123 & bitmap->mask; 124 obj = find_aligned_range(bitmap->table, 0, bitmap->max, 125 cnt, align, skip_mask); 126 } 127 128 if (obj < bitmap->max) { 129 bitmap_set(bitmap->table, obj, cnt); 130 if (obj == bitmap->last) { 131 bitmap->last = (obj + cnt); 132 if (bitmap->last >= bitmap->max) 133 bitmap->last = 0; 134 } 135 obj |= bitmap->top; 136 } else 137 obj = -1; 138 139 if (obj != -1) 140 bitmap->avail -= cnt; 141 142 spin_unlock(&bitmap->lock); 143 144 return obj; 145 } 146 147 u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) 148 { 149 return bitmap->avail; 150 } 151 152 static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj) 153 { 154 return obj & (bitmap->max + bitmap->reserved_top - 1); 155 } 156 157 void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, 158 int use_rr) 159 { 160 obj &= bitmap->max + bitmap->reserved_top - 1; 161 162 spin_lock(&bitmap->lock); 163 if (!use_rr) { 164 bitmap->last = min(bitmap->last, obj); 165 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 166 & bitmap->mask; 167 } 168 bitmap_clear(bitmap->table, obj, cnt); 169 bitmap->avail += cnt; 170 spin_unlock(&bitmap->lock); 171 } 172 173 int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, 174 u32 reserved_bot, u32 reserved_top) 175 { 176 /* num must be a power of 2 */ 177 if (num != roundup_pow_of_two(num)) 178 return -EINVAL; 179 180 bitmap->last = 0; 181 bitmap->top = 0; 182 bitmap->max = num - reserved_top; 183 bitmap->mask = mask; 184 bitmap->reserved_top = reserved_top; 185 bitmap->avail = num - reserved_top - reserved_bot; 186 bitmap->effective_len = bitmap->avail; 187 spin_lock_init(&bitmap->lock); 188 bitmap->table = kcalloc(BITS_TO_LONGS(bitmap->max), sizeof(long), 189 GFP_KERNEL); 190 if (!bitmap->table) 191 return -ENOMEM; 192 193 bitmap_set(bitmap->table, 0, reserved_bot); 194 195 return 0; 196 } 197 198 void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) 199 { 200 kfree(bitmap->table); 201 } 202 203 struct mlx4_zone_allocator { 204 struct list_head entries; 205 struct list_head prios; 206 u32 last_uid; 207 u32 mask; 208 /* protect the zone_allocator from concurrent accesses */ 209 spinlock_t lock; 210 enum mlx4_zone_alloc_flags flags; 211 }; 212 213 struct mlx4_zone_entry { 214 struct list_head list; 215 struct list_head prio_list; 216 u32 uid; 217 struct mlx4_zone_allocator *allocator; 218 struct mlx4_bitmap *bitmap; 219 int use_rr; 220 int priority; 221 int offset; 222 enum mlx4_zone_flags flags; 223 }; 224 225 struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags) 226 { 227 struct mlx4_zone_allocator *zones = kmalloc(sizeof(*zones), GFP_KERNEL); 228 229 if (NULL == zones) 230 return NULL; 231 232 INIT_LIST_HEAD(&zones->entries); 233 INIT_LIST_HEAD(&zones->prios); 234 spin_lock_init(&zones->lock); 235 zones->last_uid = 0; 236 zones->mask = 0; 237 zones->flags = flags; 238 239 return zones; 240 } 241 242 int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc, 243 struct mlx4_bitmap *bitmap, 244 u32 flags, 245 int priority, 246 int offset, 247 u32 *puid) 248 { 249 u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1); 250 struct mlx4_zone_entry *it; 251 struct mlx4_zone_entry *zone = kmalloc(sizeof(*zone), GFP_KERNEL); 252 253 if (NULL == zone) 254 return -ENOMEM; 255 256 zone->flags = flags; 257 zone->bitmap = bitmap; 258 zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0; 259 zone->priority = priority; 260 zone->offset = offset; 261 262 spin_lock(&zone_alloc->lock); 263 264 zone->uid = zone_alloc->last_uid++; 265 zone->allocator = zone_alloc; 266 267 if (zone_alloc->mask < mask) 268 zone_alloc->mask = mask; 269 270 list_for_each_entry(it, &zone_alloc->prios, prio_list) 271 if (it->priority >= priority) 272 break; 273 274 if (&it->prio_list == &zone_alloc->prios || it->priority > priority) 275 list_add_tail(&zone->prio_list, &it->prio_list); 276 list_add_tail(&zone->list, &it->list); 277 278 spin_unlock(&zone_alloc->lock); 279 280 *puid = zone->uid; 281 282 return 0; 283 } 284 285 /* Should be called under a lock */ 286 static void __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry) 287 { 288 struct mlx4_zone_allocator *zone_alloc = entry->allocator; 289 290 if (!list_empty(&entry->prio_list)) { 291 /* Check if we need to add an alternative node to the prio list */ 292 if (!list_is_last(&entry->list, &zone_alloc->entries)) { 293 struct mlx4_zone_entry *next = list_first_entry(&entry->list, 294 typeof(*next), 295 list); 296 297 if (next->priority == entry->priority) 298 list_add_tail(&next->prio_list, &entry->prio_list); 299 } 300 301 list_del(&entry->prio_list); 302 } 303 304 list_del(&entry->list); 305 306 if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) { 307 u32 mask = 0; 308 struct mlx4_zone_entry *it; 309 310 list_for_each_entry(it, &zone_alloc->prios, prio_list) { 311 u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1); 312 313 if (mask < cur_mask) 314 mask = cur_mask; 315 } 316 zone_alloc->mask = mask; 317 } 318 } 319 320 void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc) 321 { 322 struct mlx4_zone_entry *zone, *tmp; 323 324 spin_lock(&zone_alloc->lock); 325 326 list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) { 327 list_del(&zone->list); 328 list_del(&zone->prio_list); 329 kfree(zone); 330 } 331 332 spin_unlock(&zone_alloc->lock); 333 kfree(zone_alloc); 334 } 335 336 /* Should be called under a lock */ 337 static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count, 338 int align, u32 skip_mask, u32 *puid) 339 { 340 u32 uid; 341 u32 res; 342 struct mlx4_zone_allocator *zone_alloc = zone->allocator; 343 struct mlx4_zone_entry *curr_node; 344 345 res = mlx4_bitmap_alloc_range(zone->bitmap, count, 346 align, skip_mask); 347 348 if (res != (u32)-1) { 349 res += zone->offset; 350 uid = zone->uid; 351 goto out; 352 } 353 354 list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) { 355 if (unlikely(curr_node->priority == zone->priority)) 356 break; 357 } 358 359 if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) { 360 struct mlx4_zone_entry *it = curr_node; 361 362 list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) { 363 res = mlx4_bitmap_alloc_range(it->bitmap, count, 364 align, skip_mask); 365 if (res != (u32)-1) { 366 res += it->offset; 367 uid = it->uid; 368 goto out; 369 } 370 } 371 } 372 373 if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) { 374 struct mlx4_zone_entry *it = curr_node; 375 376 list_for_each_entry_from(it, &zone_alloc->entries, list) { 377 if (unlikely(it == zone)) 378 continue; 379 380 if (unlikely(it->priority != curr_node->priority)) 381 break; 382 383 res = mlx4_bitmap_alloc_range(it->bitmap, count, 384 align, skip_mask); 385 if (res != (u32)-1) { 386 res += it->offset; 387 uid = it->uid; 388 goto out; 389 } 390 } 391 } 392 393 if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) { 394 if (list_is_last(&curr_node->prio_list, &zone_alloc->prios)) 395 goto out; 396 397 curr_node = list_first_entry(&curr_node->prio_list, 398 typeof(*curr_node), 399 prio_list); 400 401 list_for_each_entry_from(curr_node, &zone_alloc->entries, list) { 402 res = mlx4_bitmap_alloc_range(curr_node->bitmap, count, 403 align, skip_mask); 404 if (res != (u32)-1) { 405 res += curr_node->offset; 406 uid = curr_node->uid; 407 goto out; 408 } 409 } 410 } 411 412 out: 413 if (NULL != puid && res != (u32)-1) 414 *puid = uid; 415 return res; 416 } 417 418 /* Should be called under a lock */ 419 static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj, 420 u32 count) 421 { 422 mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr); 423 } 424 425 /* Should be called under a lock */ 426 static struct mlx4_zone_entry *__mlx4_find_zone_by_uid( 427 struct mlx4_zone_allocator *zones, u32 uid) 428 { 429 struct mlx4_zone_entry *zone; 430 431 list_for_each_entry(zone, &zones->entries, list) { 432 if (zone->uid == uid) 433 return zone; 434 } 435 436 return NULL; 437 } 438 439 struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid) 440 { 441 struct mlx4_zone_entry *zone; 442 struct mlx4_bitmap *bitmap; 443 444 spin_lock(&zones->lock); 445 446 zone = __mlx4_find_zone_by_uid(zones, uid); 447 448 bitmap = zone == NULL ? NULL : zone->bitmap; 449 450 spin_unlock(&zones->lock); 451 452 return bitmap; 453 } 454 455 int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid) 456 { 457 struct mlx4_zone_entry *zone; 458 int res = 0; 459 460 spin_lock(&zones->lock); 461 462 zone = __mlx4_find_zone_by_uid(zones, uid); 463 464 if (NULL == zone) { 465 res = -1; 466 goto out; 467 } 468 469 __mlx4_zone_remove_one_entry(zone); 470 471 out: 472 spin_unlock(&zones->lock); 473 kfree(zone); 474 475 return res; 476 } 477 478 /* Should be called under a lock */ 479 static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique( 480 struct mlx4_zone_allocator *zones, u32 obj) 481 { 482 struct mlx4_zone_entry *zone, *zone_candidate = NULL; 483 u32 dist = (u32)-1; 484 485 /* Search for the smallest zone that this obj could be 486 * allocated from. This is done in order to handle 487 * situations when small bitmaps are allocated from bigger 488 * bitmaps (and the allocated space is marked as reserved in 489 * the bigger bitmap. 490 */ 491 list_for_each_entry(zone, &zones->entries, list) { 492 if (obj >= zone->offset) { 493 u32 mobj = (obj - zone->offset) & zones->mask; 494 495 if (mobj < zone->bitmap->max) { 496 u32 curr_dist = zone->bitmap->effective_len; 497 498 if (curr_dist < dist) { 499 dist = curr_dist; 500 zone_candidate = zone; 501 } 502 } 503 } 504 } 505 506 return zone_candidate; 507 } 508 509 u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count, 510 int align, u32 skip_mask, u32 *puid) 511 { 512 struct mlx4_zone_entry *zone; 513 int res = -1; 514 515 spin_lock(&zones->lock); 516 517 zone = __mlx4_find_zone_by_uid(zones, uid); 518 519 if (NULL == zone) 520 goto out; 521 522 res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid); 523 524 out: 525 spin_unlock(&zones->lock); 526 527 return res; 528 } 529 530 u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count) 531 { 532 struct mlx4_zone_entry *zone; 533 int res = 0; 534 535 spin_lock(&zones->lock); 536 537 zone = __mlx4_find_zone_by_uid(zones, uid); 538 539 if (NULL == zone) { 540 res = -1; 541 goto out; 542 } 543 544 __mlx4_free_from_zone(zone, obj, count); 545 546 out: 547 spin_unlock(&zones->lock); 548 549 return res; 550 } 551 552 u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count) 553 { 554 struct mlx4_zone_entry *zone; 555 int res; 556 557 if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP)) 558 return -EFAULT; 559 560 spin_lock(&zones->lock); 561 562 zone = __mlx4_find_zone_by_uid_unique(zones, obj); 563 564 if (NULL == zone) { 565 res = -1; 566 goto out; 567 } 568 569 __mlx4_free_from_zone(zone, obj, count); 570 res = 0; 571 572 out: 573 spin_unlock(&zones->lock); 574 575 return res; 576 } 577 578 static int mlx4_buf_direct_alloc(struct mlx4_dev *dev, int size, 579 struct mlx4_buf *buf) 580 { 581 dma_addr_t t; 582 583 buf->nbufs = 1; 584 buf->npages = 1; 585 buf->page_shift = get_order(size) + PAGE_SHIFT; 586 buf->direct.buf = 587 dma_zalloc_coherent(&dev->persist->pdev->dev, 588 size, &t, GFP_KERNEL); 589 if (!buf->direct.buf) 590 return -ENOMEM; 591 592 buf->direct.map = t; 593 594 while (t & ((1 << buf->page_shift) - 1)) { 595 --buf->page_shift; 596 buf->npages *= 2; 597 } 598 599 return 0; 600 } 601 602 /* Handling for queue buffers -- we allocate a bunch of memory and 603 * register it in a memory region at HCA virtual address 0. If the 604 * requested size is > max_direct, we split the allocation into 605 * multiple pages, so we don't require too much contiguous memory. 606 */ 607 int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 608 struct mlx4_buf *buf) 609 { 610 if (size <= max_direct) { 611 return mlx4_buf_direct_alloc(dev, size, buf); 612 } else { 613 dma_addr_t t; 614 int i; 615 616 buf->direct.buf = NULL; 617 buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 618 buf->npages = buf->nbufs; 619 buf->page_shift = PAGE_SHIFT; 620 buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 621 GFP_KERNEL); 622 if (!buf->page_list) 623 return -ENOMEM; 624 625 for (i = 0; i < buf->nbufs; ++i) { 626 buf->page_list[i].buf = 627 dma_zalloc_coherent(&dev->persist->pdev->dev, 628 PAGE_SIZE, &t, GFP_KERNEL); 629 if (!buf->page_list[i].buf) 630 goto err_free; 631 632 buf->page_list[i].map = t; 633 } 634 } 635 636 return 0; 637 638 err_free: 639 mlx4_buf_free(dev, size, buf); 640 641 return -ENOMEM; 642 } 643 EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 644 645 void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) 646 { 647 if (buf->nbufs == 1) { 648 dma_free_coherent(&dev->persist->pdev->dev, size, 649 buf->direct.buf, buf->direct.map); 650 } else { 651 int i; 652 653 for (i = 0; i < buf->nbufs; ++i) 654 if (buf->page_list[i].buf) 655 dma_free_coherent(&dev->persist->pdev->dev, 656 PAGE_SIZE, 657 buf->page_list[i].buf, 658 buf->page_list[i].map); 659 kfree(buf->page_list); 660 } 661 } 662 EXPORT_SYMBOL_GPL(mlx4_buf_free); 663 664 static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) 665 { 666 struct mlx4_db_pgdir *pgdir; 667 668 pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); 669 if (!pgdir) 670 return NULL; 671 672 bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 673 pgdir->bits[0] = pgdir->order0; 674 pgdir->bits[1] = pgdir->order1; 675 pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, 676 &pgdir->db_dma, GFP_KERNEL); 677 if (!pgdir->db_page) { 678 kfree(pgdir); 679 return NULL; 680 } 681 682 return pgdir; 683 } 684 685 static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 686 struct mlx4_db *db, int order) 687 { 688 int o; 689 int i; 690 691 for (o = order; o <= 1; ++o) { 692 i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 693 if (i < MLX4_DB_PER_PAGE >> o) 694 goto found; 695 } 696 697 return -ENOMEM; 698 699 found: 700 clear_bit(i, pgdir->bits[o]); 701 702 i <<= o; 703 704 if (o > order) 705 set_bit(i ^ 1, pgdir->bits[order]); 706 707 db->u.pgdir = pgdir; 708 db->index = i; 709 db->db = pgdir->db_page + db->index; 710 db->dma = pgdir->db_dma + db->index * 4; 711 db->order = order; 712 713 return 0; 714 } 715 716 int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) 717 { 718 struct mlx4_priv *priv = mlx4_priv(dev); 719 struct mlx4_db_pgdir *pgdir; 720 int ret = 0; 721 722 mutex_lock(&priv->pgdir_mutex); 723 724 list_for_each_entry(pgdir, &priv->pgdir_list, list) 725 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 726 goto out; 727 728 pgdir = mlx4_alloc_db_pgdir(&dev->persist->pdev->dev); 729 if (!pgdir) { 730 ret = -ENOMEM; 731 goto out; 732 } 733 734 list_add(&pgdir->list, &priv->pgdir_list); 735 736 /* This should never fail -- we just allocated an empty page: */ 737 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); 738 739 out: 740 mutex_unlock(&priv->pgdir_mutex); 741 742 return ret; 743 } 744 EXPORT_SYMBOL_GPL(mlx4_db_alloc); 745 746 void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) 747 { 748 struct mlx4_priv *priv = mlx4_priv(dev); 749 int o; 750 int i; 751 752 mutex_lock(&priv->pgdir_mutex); 753 754 o = db->order; 755 i = db->index; 756 757 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 758 clear_bit(i ^ 1, db->u.pgdir->order0); 759 ++o; 760 } 761 i >>= o; 762 set_bit(i, db->u.pgdir->bits[o]); 763 764 if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 765 dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, 766 db->u.pgdir->db_page, db->u.pgdir->db_dma); 767 list_del(&db->u.pgdir->list); 768 kfree(db->u.pgdir); 769 } 770 771 mutex_unlock(&priv->pgdir_mutex); 772 } 773 EXPORT_SYMBOL_GPL(mlx4_db_free); 774 775 int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 776 int size) 777 { 778 int err; 779 780 err = mlx4_db_alloc(dev, &wqres->db, 1); 781 if (err) 782 return err; 783 784 *wqres->db.db = 0; 785 786 err = mlx4_buf_direct_alloc(dev, size, &wqres->buf); 787 if (err) 788 goto err_db; 789 790 err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 791 &wqres->mtt); 792 if (err) 793 goto err_buf; 794 795 err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf); 796 if (err) 797 goto err_mtt; 798 799 return 0; 800 801 err_mtt: 802 mlx4_mtt_cleanup(dev, &wqres->mtt); 803 err_buf: 804 mlx4_buf_free(dev, size, &wqres->buf); 805 err_db: 806 mlx4_db_free(dev, &wqres->db); 807 808 return err; 809 } 810 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 811 812 void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 813 int size) 814 { 815 mlx4_mtt_cleanup(dev, &wqres->mtt); 816 mlx4_buf_free(dev, size, &wqres->buf); 817 mlx4_db_free(dev, &wqres->db); 818 } 819 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); 820