1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ 2 /* 3 * Copyright (c) 2017 Hisilicon Limited. 4 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 5 */ 6 7 #include <linux/platform_device.h> 8 #include <rdma/ib_umem.h> 9 #include "hns_roce_device.h" 10 11 int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, 12 struct hns_roce_db *db) 13 { 14 struct hns_roce_user_db_page *page; 15 int ret = 0; 16 17 mutex_lock(&context->page_mutex); 18 19 list_for_each_entry(page, &context->page_list, list) 20 if (page->user_virt == (virt & PAGE_MASK)) 21 goto found; 22 23 page = kmalloc(sizeof(*page), GFP_KERNEL); 24 if (!page) { 25 ret = -ENOMEM; 26 goto out; 27 } 28 29 refcount_set(&page->refcount, 1); 30 page->user_virt = (virt & PAGE_MASK); 31 page->umem = ib_umem_get(&context->ibucontext, virt & PAGE_MASK, 32 PAGE_SIZE, 0, 0); 33 if (IS_ERR(page->umem)) { 34 ret = PTR_ERR(page->umem); 35 kfree(page); 36 goto out; 37 } 38 39 list_add(&page->list, &context->page_list); 40 41 found: 42 db->dma = sg_dma_address(page->umem->sg_head.sgl) + 43 (virt & ~PAGE_MASK); 44 db->u.user_page = page; 45 refcount_inc(&page->refcount); 46 47 out: 48 mutex_unlock(&context->page_mutex); 49 50 return ret; 51 } 52 EXPORT_SYMBOL(hns_roce_db_map_user); 53 54 void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, 55 struct hns_roce_db *db) 56 { 57 mutex_lock(&context->page_mutex); 58 59 refcount_dec(&db->u.user_page->refcount); 60 if (refcount_dec_if_one(&db->u.user_page->refcount)) { 61 list_del(&db->u.user_page->list); 62 ib_umem_release(db->u.user_page->umem); 63 kfree(db->u.user_page); 64 } 65 66 mutex_unlock(&context->page_mutex); 67 } 68 EXPORT_SYMBOL(hns_roce_db_unmap_user); 69 70 static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( 71 struct device *dma_device) 72 { 73 struct hns_roce_db_pgdir *pgdir; 74 75 pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); 76 if (!pgdir) 77 return NULL; 78 79 bitmap_fill(pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2); 80 pgdir->bits[0] = pgdir->order0; 81 pgdir->bits[1] = pgdir->order1; 82 pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE, 83 &pgdir->db_dma, GFP_KERNEL); 84 if (!pgdir->page) { 85 kfree(pgdir); 86 return NULL; 87 } 88 89 return pgdir; 90 } 91 92 static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir, 93 struct hns_roce_db *db, int order) 94 { 95 int o; 96 int i; 97 98 for (o = order; o <= 1; ++o) { 99 i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o); 100 if (i < HNS_ROCE_DB_PER_PAGE >> o) 101 goto found; 102 } 103 104 return -ENOMEM; 105 106 found: 107 clear_bit(i, pgdir->bits[o]); 108 109 i <<= o; 110 111 if (o > order) 112 set_bit(i ^ 1, pgdir->bits[order]); 113 114 db->u.pgdir = pgdir; 115 db->index = i; 116 db->db_record = pgdir->page + db->index; 117 db->dma = pgdir->db_dma + db->index * 4; 118 db->order = order; 119 120 return 0; 121 } 122 123 int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, 124 int order) 125 { 126 struct hns_roce_db_pgdir *pgdir; 127 int ret = 0; 128 129 mutex_lock(&hr_dev->pgdir_mutex); 130 131 list_for_each_entry(pgdir, &hr_dev->pgdir_list, list) 132 if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order)) 133 goto out; 134 135 pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev); 136 if (!pgdir) { 137 ret = -ENOMEM; 138 goto out; 139 } 140 141 list_add(&pgdir->list, &hr_dev->pgdir_list); 142 143 /* This should never fail -- we just allocated an empty page: */ 144 WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order)); 145 146 out: 147 mutex_unlock(&hr_dev->pgdir_mutex); 148 149 return ret; 150 } 151 EXPORT_SYMBOL_GPL(hns_roce_alloc_db); 152 153 void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) 154 { 155 int o; 156 int i; 157 158 mutex_lock(&hr_dev->pgdir_mutex); 159 160 o = db->order; 161 i = db->index; 162 163 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 164 clear_bit(i ^ 1, db->u.pgdir->order0); 165 ++o; 166 } 167 168 i >>= o; 169 set_bit(i, db->u.pgdir->bits[o]); 170 171 if (bitmap_full(db->u.pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2)) { 172 dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, 173 db->u.pgdir->db_dma); 174 list_del(&db->u.pgdir->list); 175 kfree(db->u.pgdir); 176 } 177 178 mutex_unlock(&hr_dev->pgdir_mutex); 179 } 180 EXPORT_SYMBOL_GPL(hns_roce_free_db); 181