15a2cc190SJeff Kirsher /* 25a2cc190SJeff Kirsher * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 35a2cc190SJeff Kirsher * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 45a2cc190SJeff Kirsher * 55a2cc190SJeff Kirsher * This software is available to you under a choice of one of two 65a2cc190SJeff Kirsher * licenses. You may choose to be licensed under the terms of the GNU 75a2cc190SJeff Kirsher * General Public License (GPL) Version 2, available from the file 85a2cc190SJeff Kirsher * COPYING in the main directory of this source tree, or the 95a2cc190SJeff Kirsher * OpenIB.org BSD license below: 105a2cc190SJeff Kirsher * 115a2cc190SJeff Kirsher * Redistribution and use in source and binary forms, with or 125a2cc190SJeff Kirsher * without modification, are permitted provided that the following 135a2cc190SJeff Kirsher * conditions are met: 145a2cc190SJeff Kirsher * 155a2cc190SJeff Kirsher * - Redistributions of source code must retain the above 165a2cc190SJeff Kirsher * copyright notice, this list of conditions and the following 175a2cc190SJeff Kirsher * disclaimer. 185a2cc190SJeff Kirsher * 195a2cc190SJeff Kirsher * - Redistributions in binary form must reproduce the above 205a2cc190SJeff Kirsher * copyright notice, this list of conditions and the following 215a2cc190SJeff Kirsher * disclaimer in the documentation and/or other materials 225a2cc190SJeff Kirsher * provided with the distribution. 235a2cc190SJeff Kirsher * 245a2cc190SJeff Kirsher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 255a2cc190SJeff Kirsher * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 265a2cc190SJeff Kirsher * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 275a2cc190SJeff Kirsher * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 285a2cc190SJeff Kirsher * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 295a2cc190SJeff Kirsher * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 305a2cc190SJeff Kirsher * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 315a2cc190SJeff Kirsher * SOFTWARE. 325a2cc190SJeff Kirsher */ 335a2cc190SJeff Kirsher 345a2cc190SJeff Kirsher #include <linux/errno.h> 355a2cc190SJeff Kirsher #include <linux/slab.h> 365a2cc190SJeff Kirsher #include <linux/mm.h> 375a2cc190SJeff Kirsher #include <linux/bitmap.h> 385a2cc190SJeff Kirsher #include <linux/dma-mapping.h> 395a2cc190SJeff Kirsher #include <linux/vmalloc.h> 405a2cc190SJeff Kirsher 415a2cc190SJeff Kirsher #include "mlx4.h" 425a2cc190SJeff Kirsher 435a2cc190SJeff Kirsher u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) 445a2cc190SJeff Kirsher { 455a2cc190SJeff Kirsher u32 obj; 465a2cc190SJeff Kirsher 475a2cc190SJeff Kirsher spin_lock(&bitmap->lock); 485a2cc190SJeff Kirsher 495a2cc190SJeff Kirsher obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); 505a2cc190SJeff Kirsher if (obj >= bitmap->max) { 515a2cc190SJeff Kirsher bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 525a2cc190SJeff Kirsher & bitmap->mask; 535a2cc190SJeff Kirsher obj = find_first_zero_bit(bitmap->table, bitmap->max); 545a2cc190SJeff Kirsher } 555a2cc190SJeff Kirsher 565a2cc190SJeff Kirsher if (obj < bitmap->max) { 575a2cc190SJeff Kirsher set_bit(obj, bitmap->table); 585a2cc190SJeff Kirsher bitmap->last = (obj + 1); 595a2cc190SJeff Kirsher if (bitmap->last == bitmap->max) 605a2cc190SJeff Kirsher bitmap->last = 0; 615a2cc190SJeff Kirsher obj |= bitmap->top; 625a2cc190SJeff Kirsher } else 635a2cc190SJeff Kirsher obj = -1; 645a2cc190SJeff Kirsher 655a2cc190SJeff Kirsher if (obj != -1) 665a2cc190SJeff Kirsher --bitmap->avail; 675a2cc190SJeff Kirsher 685a2cc190SJeff Kirsher spin_unlock(&bitmap->lock); 695a2cc190SJeff Kirsher 705a2cc190SJeff Kirsher return obj; 715a2cc190SJeff Kirsher } 725a2cc190SJeff Kirsher 735a2cc190SJeff Kirsher void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj) 745a2cc190SJeff Kirsher { 755a2cc190SJeff Kirsher mlx4_bitmap_free_range(bitmap, obj, 1); 765a2cc190SJeff Kirsher } 775a2cc190SJeff Kirsher 785a2cc190SJeff Kirsher u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align) 795a2cc190SJeff Kirsher { 805a2cc190SJeff Kirsher u32 obj; 815a2cc190SJeff Kirsher 825a2cc190SJeff Kirsher if (likely(cnt == 1 && align == 1)) 835a2cc190SJeff Kirsher return mlx4_bitmap_alloc(bitmap); 845a2cc190SJeff Kirsher 855a2cc190SJeff Kirsher spin_lock(&bitmap->lock); 865a2cc190SJeff Kirsher 875a2cc190SJeff Kirsher obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 885a2cc190SJeff Kirsher bitmap->last, cnt, align - 1); 895a2cc190SJeff Kirsher if (obj >= bitmap->max) { 905a2cc190SJeff Kirsher bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 915a2cc190SJeff Kirsher & bitmap->mask; 925a2cc190SJeff Kirsher obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 935a2cc190SJeff Kirsher 0, cnt, align - 1); 945a2cc190SJeff Kirsher } 955a2cc190SJeff Kirsher 965a2cc190SJeff Kirsher if (obj < bitmap->max) { 975a2cc190SJeff Kirsher bitmap_set(bitmap->table, obj, cnt); 985a2cc190SJeff Kirsher if (obj == bitmap->last) { 995a2cc190SJeff Kirsher bitmap->last = (obj + cnt); 1005a2cc190SJeff Kirsher if (bitmap->last >= bitmap->max) 1015a2cc190SJeff Kirsher bitmap->last = 0; 1025a2cc190SJeff Kirsher } 1035a2cc190SJeff Kirsher obj |= bitmap->top; 1045a2cc190SJeff Kirsher } else 1055a2cc190SJeff Kirsher obj = -1; 1065a2cc190SJeff Kirsher 1075a2cc190SJeff Kirsher if (obj != -1) 1085a2cc190SJeff Kirsher bitmap->avail -= cnt; 1095a2cc190SJeff Kirsher 1105a2cc190SJeff Kirsher spin_unlock(&bitmap->lock); 1115a2cc190SJeff Kirsher 1125a2cc190SJeff Kirsher return obj; 1135a2cc190SJeff Kirsher } 1145a2cc190SJeff Kirsher 1155a2cc190SJeff Kirsher u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) 1165a2cc190SJeff Kirsher { 1175a2cc190SJeff Kirsher return bitmap->avail; 1185a2cc190SJeff Kirsher } 1195a2cc190SJeff Kirsher 1205a2cc190SJeff Kirsher void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt) 1215a2cc190SJeff Kirsher { 1225a2cc190SJeff Kirsher obj &= bitmap->max + bitmap->reserved_top - 1; 1235a2cc190SJeff Kirsher 1245a2cc190SJeff Kirsher spin_lock(&bitmap->lock); 1255a2cc190SJeff Kirsher bitmap_clear(bitmap->table, obj, cnt); 1265a2cc190SJeff Kirsher bitmap->last = min(bitmap->last, obj); 1275a2cc190SJeff Kirsher bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) 1285a2cc190SJeff Kirsher & bitmap->mask; 1295a2cc190SJeff Kirsher bitmap->avail += cnt; 1305a2cc190SJeff Kirsher spin_unlock(&bitmap->lock); 1315a2cc190SJeff Kirsher } 1325a2cc190SJeff Kirsher 1335a2cc190SJeff Kirsher int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, 1345a2cc190SJeff Kirsher u32 reserved_bot, u32 reserved_top) 1355a2cc190SJeff Kirsher { 1365a2cc190SJeff Kirsher /* num must be a power of 2 */ 1375a2cc190SJeff Kirsher if (num != roundup_pow_of_two(num)) 1385a2cc190SJeff Kirsher return -EINVAL; 1395a2cc190SJeff Kirsher 1405a2cc190SJeff Kirsher bitmap->last = 0; 1415a2cc190SJeff Kirsher bitmap->top = 0; 1425a2cc190SJeff Kirsher bitmap->max = num - reserved_top; 1435a2cc190SJeff Kirsher bitmap->mask = mask; 1445a2cc190SJeff Kirsher bitmap->reserved_top = reserved_top; 1455a2cc190SJeff Kirsher bitmap->avail = num - reserved_top - reserved_bot; 1465a2cc190SJeff Kirsher spin_lock_init(&bitmap->lock); 1475a2cc190SJeff Kirsher bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * 1485a2cc190SJeff Kirsher sizeof (long), GFP_KERNEL); 1495a2cc190SJeff Kirsher if (!bitmap->table) 1505a2cc190SJeff Kirsher return -ENOMEM; 1515a2cc190SJeff Kirsher 1525a2cc190SJeff Kirsher bitmap_set(bitmap->table, 0, reserved_bot); 1535a2cc190SJeff Kirsher 1545a2cc190SJeff Kirsher return 0; 1555a2cc190SJeff Kirsher } 1565a2cc190SJeff Kirsher 1575a2cc190SJeff Kirsher void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) 1585a2cc190SJeff Kirsher { 1595a2cc190SJeff Kirsher kfree(bitmap->table); 1605a2cc190SJeff Kirsher } 1615a2cc190SJeff Kirsher 1625a2cc190SJeff Kirsher /* 1635a2cc190SJeff Kirsher * Handling for queue buffers -- we allocate a bunch of memory and 1645a2cc190SJeff Kirsher * register it in a memory region at HCA virtual address 0. If the 1655a2cc190SJeff Kirsher * requested size is > max_direct, we split the allocation into 1665a2cc190SJeff Kirsher * multiple pages, so we don't require too much contiguous memory. 1675a2cc190SJeff Kirsher */ 1685a2cc190SJeff Kirsher 1695a2cc190SJeff Kirsher int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, 1705a2cc190SJeff Kirsher struct mlx4_buf *buf) 1715a2cc190SJeff Kirsher { 1725a2cc190SJeff Kirsher dma_addr_t t; 1735a2cc190SJeff Kirsher 1745a2cc190SJeff Kirsher if (size <= max_direct) { 1755a2cc190SJeff Kirsher buf->nbufs = 1; 1765a2cc190SJeff Kirsher buf->npages = 1; 1775a2cc190SJeff Kirsher buf->page_shift = get_order(size) + PAGE_SHIFT; 1785a2cc190SJeff Kirsher buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 1795a2cc190SJeff Kirsher size, &t, GFP_KERNEL); 1805a2cc190SJeff Kirsher if (!buf->direct.buf) 1815a2cc190SJeff Kirsher return -ENOMEM; 1825a2cc190SJeff Kirsher 1835a2cc190SJeff Kirsher buf->direct.map = t; 1845a2cc190SJeff Kirsher 1855a2cc190SJeff Kirsher while (t & ((1 << buf->page_shift) - 1)) { 1865a2cc190SJeff Kirsher --buf->page_shift; 1875a2cc190SJeff Kirsher buf->npages *= 2; 1885a2cc190SJeff Kirsher } 1895a2cc190SJeff Kirsher 1905a2cc190SJeff Kirsher memset(buf->direct.buf, 0, size); 1915a2cc190SJeff Kirsher } else { 1925a2cc190SJeff Kirsher int i; 1935a2cc190SJeff Kirsher 1945a2cc190SJeff Kirsher buf->direct.buf = NULL; 1955a2cc190SJeff Kirsher buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; 1965a2cc190SJeff Kirsher buf->npages = buf->nbufs; 1975a2cc190SJeff Kirsher buf->page_shift = PAGE_SHIFT; 1985a2cc190SJeff Kirsher buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), 1995a2cc190SJeff Kirsher GFP_KERNEL); 2005a2cc190SJeff Kirsher if (!buf->page_list) 2015a2cc190SJeff Kirsher return -ENOMEM; 2025a2cc190SJeff Kirsher 2035a2cc190SJeff Kirsher for (i = 0; i < buf->nbufs; ++i) { 2045a2cc190SJeff Kirsher buf->page_list[i].buf = 2055a2cc190SJeff Kirsher dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 2065a2cc190SJeff Kirsher &t, GFP_KERNEL); 2075a2cc190SJeff Kirsher if (!buf->page_list[i].buf) 2085a2cc190SJeff Kirsher goto err_free; 2095a2cc190SJeff Kirsher 2105a2cc190SJeff Kirsher buf->page_list[i].map = t; 2115a2cc190SJeff Kirsher 2125a2cc190SJeff Kirsher memset(buf->page_list[i].buf, 0, PAGE_SIZE); 2135a2cc190SJeff Kirsher } 2145a2cc190SJeff Kirsher 2155a2cc190SJeff Kirsher if (BITS_PER_LONG == 64) { 2165a2cc190SJeff Kirsher struct page **pages; 2175a2cc190SJeff Kirsher pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL); 2185a2cc190SJeff Kirsher if (!pages) 2195a2cc190SJeff Kirsher goto err_free; 2205a2cc190SJeff Kirsher for (i = 0; i < buf->nbufs; ++i) 2215a2cc190SJeff Kirsher pages[i] = virt_to_page(buf->page_list[i].buf); 2225a2cc190SJeff Kirsher buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); 2235a2cc190SJeff Kirsher kfree(pages); 2245a2cc190SJeff Kirsher if (!buf->direct.buf) 2255a2cc190SJeff Kirsher goto err_free; 2265a2cc190SJeff Kirsher } 2275a2cc190SJeff Kirsher } 2285a2cc190SJeff Kirsher 2295a2cc190SJeff Kirsher return 0; 2305a2cc190SJeff Kirsher 2315a2cc190SJeff Kirsher err_free: 2325a2cc190SJeff Kirsher mlx4_buf_free(dev, size, buf); 2335a2cc190SJeff Kirsher 2345a2cc190SJeff Kirsher return -ENOMEM; 2355a2cc190SJeff Kirsher } 2365a2cc190SJeff Kirsher EXPORT_SYMBOL_GPL(mlx4_buf_alloc); 2375a2cc190SJeff Kirsher 2385a2cc190SJeff Kirsher void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) 2395a2cc190SJeff Kirsher { 2405a2cc190SJeff Kirsher int i; 2415a2cc190SJeff Kirsher 2425a2cc190SJeff Kirsher if (buf->nbufs == 1) 2435a2cc190SJeff Kirsher dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 2445a2cc190SJeff Kirsher buf->direct.map); 2455a2cc190SJeff Kirsher else { 2465a2cc190SJeff Kirsher if (BITS_PER_LONG == 64 && buf->direct.buf) 2475a2cc190SJeff Kirsher vunmap(buf->direct.buf); 2485a2cc190SJeff Kirsher 2495a2cc190SJeff Kirsher for (i = 0; i < buf->nbufs; ++i) 2505a2cc190SJeff Kirsher if (buf->page_list[i].buf) 2515a2cc190SJeff Kirsher dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 2525a2cc190SJeff Kirsher buf->page_list[i].buf, 2535a2cc190SJeff Kirsher buf->page_list[i].map); 2545a2cc190SJeff Kirsher kfree(buf->page_list); 2555a2cc190SJeff Kirsher } 2565a2cc190SJeff Kirsher } 2575a2cc190SJeff Kirsher EXPORT_SYMBOL_GPL(mlx4_buf_free); 2585a2cc190SJeff Kirsher 2595a2cc190SJeff Kirsher static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) 2605a2cc190SJeff Kirsher { 2615a2cc190SJeff Kirsher struct mlx4_db_pgdir *pgdir; 2625a2cc190SJeff Kirsher 2635a2cc190SJeff Kirsher pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL); 2645a2cc190SJeff Kirsher if (!pgdir) 2655a2cc190SJeff Kirsher return NULL; 2665a2cc190SJeff Kirsher 2675a2cc190SJeff Kirsher bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); 2685a2cc190SJeff Kirsher pgdir->bits[0] = pgdir->order0; 2695a2cc190SJeff Kirsher pgdir->bits[1] = pgdir->order1; 2705a2cc190SJeff Kirsher pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, 2715a2cc190SJeff Kirsher &pgdir->db_dma, GFP_KERNEL); 2725a2cc190SJeff Kirsher if (!pgdir->db_page) { 2735a2cc190SJeff Kirsher kfree(pgdir); 2745a2cc190SJeff Kirsher return NULL; 2755a2cc190SJeff Kirsher } 2765a2cc190SJeff Kirsher 2775a2cc190SJeff Kirsher return pgdir; 2785a2cc190SJeff Kirsher } 2795a2cc190SJeff Kirsher 2805a2cc190SJeff Kirsher static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, 2815a2cc190SJeff Kirsher struct mlx4_db *db, int order) 2825a2cc190SJeff Kirsher { 2835a2cc190SJeff Kirsher int o; 2845a2cc190SJeff Kirsher int i; 2855a2cc190SJeff Kirsher 2865a2cc190SJeff Kirsher for (o = order; o <= 1; ++o) { 2875a2cc190SJeff Kirsher i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); 2885a2cc190SJeff Kirsher if (i < MLX4_DB_PER_PAGE >> o) 2895a2cc190SJeff Kirsher goto found; 2905a2cc190SJeff Kirsher } 2915a2cc190SJeff Kirsher 2925a2cc190SJeff Kirsher return -ENOMEM; 2935a2cc190SJeff Kirsher 2945a2cc190SJeff Kirsher found: 2955a2cc190SJeff Kirsher clear_bit(i, pgdir->bits[o]); 2965a2cc190SJeff Kirsher 2975a2cc190SJeff Kirsher i <<= o; 2985a2cc190SJeff Kirsher 2995a2cc190SJeff Kirsher if (o > order) 3005a2cc190SJeff Kirsher set_bit(i ^ 1, pgdir->bits[order]); 3015a2cc190SJeff Kirsher 3025a2cc190SJeff Kirsher db->u.pgdir = pgdir; 3035a2cc190SJeff Kirsher db->index = i; 3045a2cc190SJeff Kirsher db->db = pgdir->db_page + db->index; 3055a2cc190SJeff Kirsher db->dma = pgdir->db_dma + db->index * 4; 3065a2cc190SJeff Kirsher db->order = order; 3075a2cc190SJeff Kirsher 3085a2cc190SJeff Kirsher return 0; 3095a2cc190SJeff Kirsher } 3105a2cc190SJeff Kirsher 3115a2cc190SJeff Kirsher int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) 3125a2cc190SJeff Kirsher { 3135a2cc190SJeff Kirsher struct mlx4_priv *priv = mlx4_priv(dev); 3145a2cc190SJeff Kirsher struct mlx4_db_pgdir *pgdir; 3155a2cc190SJeff Kirsher int ret = 0; 3165a2cc190SJeff Kirsher 3175a2cc190SJeff Kirsher mutex_lock(&priv->pgdir_mutex); 3185a2cc190SJeff Kirsher 3195a2cc190SJeff Kirsher list_for_each_entry(pgdir, &priv->pgdir_list, list) 3205a2cc190SJeff Kirsher if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) 3215a2cc190SJeff Kirsher goto out; 3225a2cc190SJeff Kirsher 3235a2cc190SJeff Kirsher pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev)); 3245a2cc190SJeff Kirsher if (!pgdir) { 3255a2cc190SJeff Kirsher ret = -ENOMEM; 3265a2cc190SJeff Kirsher goto out; 3275a2cc190SJeff Kirsher } 3285a2cc190SJeff Kirsher 3295a2cc190SJeff Kirsher list_add(&pgdir->list, &priv->pgdir_list); 3305a2cc190SJeff Kirsher 3315a2cc190SJeff Kirsher /* This should never fail -- we just allocated an empty page: */ 3325a2cc190SJeff Kirsher WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); 3335a2cc190SJeff Kirsher 3345a2cc190SJeff Kirsher out: 3355a2cc190SJeff Kirsher mutex_unlock(&priv->pgdir_mutex); 3365a2cc190SJeff Kirsher 3375a2cc190SJeff Kirsher return ret; 3385a2cc190SJeff Kirsher } 3395a2cc190SJeff Kirsher EXPORT_SYMBOL_GPL(mlx4_db_alloc); 3405a2cc190SJeff Kirsher 3415a2cc190SJeff Kirsher void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) 3425a2cc190SJeff Kirsher { 3435a2cc190SJeff Kirsher struct mlx4_priv *priv = mlx4_priv(dev); 3445a2cc190SJeff Kirsher int o; 3455a2cc190SJeff Kirsher int i; 3465a2cc190SJeff Kirsher 3475a2cc190SJeff Kirsher mutex_lock(&priv->pgdir_mutex); 3485a2cc190SJeff Kirsher 3495a2cc190SJeff Kirsher o = db->order; 3505a2cc190SJeff Kirsher i = db->index; 3515a2cc190SJeff Kirsher 3525a2cc190SJeff Kirsher if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 3535a2cc190SJeff Kirsher clear_bit(i ^ 1, db->u.pgdir->order0); 3545a2cc190SJeff Kirsher ++o; 3555a2cc190SJeff Kirsher } 3565a2cc190SJeff Kirsher i >>= o; 3575a2cc190SJeff Kirsher set_bit(i, db->u.pgdir->bits[o]); 3585a2cc190SJeff Kirsher 3595a2cc190SJeff Kirsher if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { 3605a2cc190SJeff Kirsher dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, 3615a2cc190SJeff Kirsher db->u.pgdir->db_page, db->u.pgdir->db_dma); 3625a2cc190SJeff Kirsher list_del(&db->u.pgdir->list); 3635a2cc190SJeff Kirsher kfree(db->u.pgdir); 3645a2cc190SJeff Kirsher } 3655a2cc190SJeff Kirsher 3665a2cc190SJeff Kirsher mutex_unlock(&priv->pgdir_mutex); 3675a2cc190SJeff Kirsher } 3685a2cc190SJeff Kirsher EXPORT_SYMBOL_GPL(mlx4_db_free); 3695a2cc190SJeff Kirsher 3705a2cc190SJeff Kirsher int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 3715a2cc190SJeff Kirsher int size, int max_direct) 3725a2cc190SJeff Kirsher { 3735a2cc190SJeff Kirsher int err; 3745a2cc190SJeff Kirsher 3755a2cc190SJeff Kirsher err = mlx4_db_alloc(dev, &wqres->db, 1); 3765a2cc190SJeff Kirsher if (err) 3775a2cc190SJeff Kirsher return err; 3785a2cc190SJeff Kirsher 3795a2cc190SJeff Kirsher *wqres->db.db = 0; 3805a2cc190SJeff Kirsher 3815a2cc190SJeff Kirsher err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf); 3825a2cc190SJeff Kirsher if (err) 3835a2cc190SJeff Kirsher goto err_db; 3845a2cc190SJeff Kirsher 3855a2cc190SJeff Kirsher err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, 3865a2cc190SJeff Kirsher &wqres->mtt); 3875a2cc190SJeff Kirsher if (err) 3885a2cc190SJeff Kirsher goto err_buf; 3895a2cc190SJeff Kirsher 3905a2cc190SJeff Kirsher err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf); 3915a2cc190SJeff Kirsher if (err) 3925a2cc190SJeff Kirsher goto err_mtt; 3935a2cc190SJeff Kirsher 3945a2cc190SJeff Kirsher return 0; 3955a2cc190SJeff Kirsher 3965a2cc190SJeff Kirsher err_mtt: 3975a2cc190SJeff Kirsher mlx4_mtt_cleanup(dev, &wqres->mtt); 3985a2cc190SJeff Kirsher err_buf: 3995a2cc190SJeff Kirsher mlx4_buf_free(dev, size, &wqres->buf); 4005a2cc190SJeff Kirsher err_db: 4015a2cc190SJeff Kirsher mlx4_db_free(dev, &wqres->db); 4025a2cc190SJeff Kirsher 4035a2cc190SJeff Kirsher return err; 4045a2cc190SJeff Kirsher } 4055a2cc190SJeff Kirsher EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); 4065a2cc190SJeff Kirsher 4075a2cc190SJeff Kirsher void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, 4085a2cc190SJeff Kirsher int size) 4095a2cc190SJeff Kirsher { 4105a2cc190SJeff Kirsher mlx4_mtt_cleanup(dev, &wqres->mtt); 4115a2cc190SJeff Kirsher mlx4_buf_free(dev, size, &wqres->buf); 4125a2cc190SJeff Kirsher mlx4_db_free(dev, &wqres->db); 4135a2cc190SJeff Kirsher } 4145a2cc190SJeff Kirsher EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); 415