1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
2 /*
3  * Copyright (c) 2017 Hisilicon Limited.
4  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5  */
6 
7 #include <rdma/ib_umem.h>
8 #include "hns_roce_device.h"
9 
10 int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
11 			 struct hns_roce_db *db)
12 {
13 	unsigned long page_addr = virt & PAGE_MASK;
14 	struct hns_roce_user_db_page *page;
15 	unsigned int offset;
16 	int ret = 0;
17 
18 	mutex_lock(&context->page_mutex);
19 
20 	list_for_each_entry(page, &context->page_list, list)
21 		if (page->user_virt == page_addr)
22 			goto found;
23 
24 	page = kmalloc(sizeof(*page), GFP_KERNEL);
25 	if (!page) {
26 		ret = -ENOMEM;
27 		goto out;
28 	}
29 
30 	refcount_set(&page->refcount, 1);
31 	page->user_virt = page_addr;
32 	page->umem = ib_umem_get(context->ibucontext.device, page_addr,
33 				 PAGE_SIZE, 0);
34 	if (IS_ERR(page->umem)) {
35 		ret = PTR_ERR(page->umem);
36 		kfree(page);
37 		goto out;
38 	}
39 
40 	list_add(&page->list, &context->page_list);
41 
42 found:
43 	offset = virt - page_addr;
44 	db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset;
45 	db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset;
46 	db->u.user_page = page;
47 	refcount_inc(&page->refcount);
48 
49 out:
50 	mutex_unlock(&context->page_mutex);
51 
52 	return ret;
53 }
54 
55 void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
56 			    struct hns_roce_db *db)
57 {
58 	mutex_lock(&context->page_mutex);
59 
60 	refcount_dec(&db->u.user_page->refcount);
61 	if (refcount_dec_if_one(&db->u.user_page->refcount)) {
62 		list_del(&db->u.user_page->list);
63 		ib_umem_release(db->u.user_page->umem);
64 		kfree(db->u.user_page);
65 	}
66 
67 	mutex_unlock(&context->page_mutex);
68 }
69 
70 static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(
71 					struct device *dma_device)
72 {
73 	struct hns_roce_db_pgdir *pgdir;
74 
75 	pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL);
76 	if (!pgdir)
77 		return NULL;
78 
79 	bitmap_fill(pgdir->order1,
80 		    HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
81 	pgdir->bits[0] = pgdir->order0;
82 	pgdir->bits[1] = pgdir->order1;
83 	pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE,
84 					 &pgdir->db_dma, GFP_KERNEL);
85 	if (!pgdir->page) {
86 		kfree(pgdir);
87 		return NULL;
88 	}
89 
90 	return pgdir;
91 }
92 
93 static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir,
94 					struct hns_roce_db *db, int order)
95 {
96 	unsigned long o;
97 	unsigned long i;
98 
99 	for (o = order; o <= 1; ++o) {
100 		i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o);
101 		if (i < HNS_ROCE_DB_PER_PAGE >> o)
102 			goto found;
103 	}
104 
105 	return -ENOMEM;
106 
107 found:
108 	clear_bit(i, pgdir->bits[o]);
109 
110 	i <<= o;
111 
112 	if (o > order)
113 		set_bit(i ^ 1, pgdir->bits[order]);
114 
115 	db->u.pgdir	= pgdir;
116 	db->index	= i;
117 	db->db_record	= pgdir->page + db->index;
118 	db->dma		= pgdir->db_dma  + db->index * HNS_ROCE_DB_UNIT_SIZE;
119 	db->order	= order;
120 
121 	return 0;
122 }
123 
124 int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,
125 		      int order)
126 {
127 	struct hns_roce_db_pgdir *pgdir;
128 	int ret = 0;
129 
130 	mutex_lock(&hr_dev->pgdir_mutex);
131 
132 	list_for_each_entry(pgdir, &hr_dev->pgdir_list, list)
133 		if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order))
134 			goto out;
135 
136 	pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev);
137 	if (!pgdir) {
138 		ret = -ENOMEM;
139 		goto out;
140 	}
141 
142 	list_add(&pgdir->list, &hr_dev->pgdir_list);
143 
144 	/* This should never fail -- we just allocated an empty page: */
145 	WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order));
146 
147 out:
148 	mutex_unlock(&hr_dev->pgdir_mutex);
149 
150 	return ret;
151 }
152 
153 void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
154 {
155 	unsigned long o;
156 	unsigned long i;
157 
158 	mutex_lock(&hr_dev->pgdir_mutex);
159 
160 	o = db->order;
161 	i = db->index;
162 
163 	if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
164 		clear_bit(i ^ 1, db->u.pgdir->order0);
165 		++o;
166 	}
167 
168 	i >>= o;
169 	set_bit(i, db->u.pgdir->bits[o]);
170 
171 	if (bitmap_full(db->u.pgdir->order1,
172 			HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) {
173 		dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page,
174 				  db->u.pgdir->db_dma);
175 		list_del(&db->u.pgdir->list);
176 		kfree(db->u.pgdir);
177 	}
178 
179 	mutex_unlock(&hr_dev->pgdir_mutex);
180 }
181