1 /*
2  * Copyright (c) 2016 Hisilicon Limited.
3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/platform_device.h>
35 #include <linux/vmalloc.h>
36 #include "hns_roce_device.h"
37 #include <rdma/ib_umem.h>
38 
39 int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj)
40 {
41 	int ret = 0;
42 
43 	spin_lock(&bitmap->lock);
44 	*obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
45 	if (*obj >= bitmap->max) {
46 		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
47 			       & bitmap->mask;
48 		*obj = find_first_zero_bit(bitmap->table, bitmap->max);
49 	}
50 
51 	if (*obj < bitmap->max) {
52 		set_bit(*obj, bitmap->table);
53 		bitmap->last = (*obj + 1);
54 		if (bitmap->last == bitmap->max)
55 			bitmap->last = 0;
56 		*obj |= bitmap->top;
57 	} else {
58 		ret = -EINVAL;
59 	}
60 
61 	spin_unlock(&bitmap->lock);
62 
63 	return ret;
64 }
65 
66 void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,
67 			  int rr)
68 {
69 	hns_roce_bitmap_free_range(bitmap, obj, 1, rr);
70 }
71 
72 int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,
73 				int align, unsigned long *obj)
74 {
75 	int ret = 0;
76 	int i;
77 
78 	if (likely(cnt == 1 && align == 1))
79 		return hns_roce_bitmap_alloc(bitmap, obj);
80 
81 	spin_lock(&bitmap->lock);
82 
83 	*obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
84 					  bitmap->last, cnt, align - 1);
85 	if (*obj >= bitmap->max) {
86 		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
87 			       & bitmap->mask;
88 		*obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, 0,
89 						  cnt, align - 1);
90 	}
91 
92 	if (*obj < bitmap->max) {
93 		for (i = 0; i < cnt; i++)
94 			set_bit(*obj + i, bitmap->table);
95 
96 		if (*obj == bitmap->last) {
97 			bitmap->last = (*obj + cnt);
98 			if (bitmap->last >= bitmap->max)
99 				bitmap->last = 0;
100 		}
101 		*obj |= bitmap->top;
102 	} else {
103 		ret = -EINVAL;
104 	}
105 
106 	spin_unlock(&bitmap->lock);
107 
108 	return ret;
109 }
110 
111 void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
112 				unsigned long obj, int cnt,
113 				int rr)
114 {
115 	int i;
116 
117 	obj &= bitmap->max + bitmap->reserved_top - 1;
118 
119 	spin_lock(&bitmap->lock);
120 	for (i = 0; i < cnt; i++)
121 		clear_bit(obj + i, bitmap->table);
122 
123 	if (!rr)
124 		bitmap->last = min(bitmap->last, obj);
125 	bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
126 		       & bitmap->mask;
127 	spin_unlock(&bitmap->lock);
128 }
129 
130 int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask,
131 			 u32 reserved_bot, u32 reserved_top)
132 {
133 	u32 i;
134 
135 	if (num != roundup_pow_of_two(num))
136 		return -EINVAL;
137 
138 	bitmap->last = 0;
139 	bitmap->top = 0;
140 	bitmap->max = num - reserved_top;
141 	bitmap->mask = mask;
142 	bitmap->reserved_top = reserved_top;
143 	spin_lock_init(&bitmap->lock);
144 	bitmap->table = kcalloc(BITS_TO_LONGS(bitmap->max), sizeof(long),
145 				GFP_KERNEL);
146 	if (!bitmap->table)
147 		return -ENOMEM;
148 
149 	for (i = 0; i < reserved_bot; ++i)
150 		set_bit(i, bitmap->table);
151 
152 	return 0;
153 }
154 
155 void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap)
156 {
157 	kfree(bitmap->table);
158 }
159 
160 void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf)
161 {
162 	struct hns_roce_buf_list *trunks;
163 	u32 i;
164 
165 	if (!buf)
166 		return;
167 
168 	trunks = buf->trunk_list;
169 	if (trunks) {
170 		buf->trunk_list = NULL;
171 		for (i = 0; i < buf->ntrunks; i++)
172 			dma_free_coherent(hr_dev->dev, 1 << buf->trunk_shift,
173 					  trunks[i].buf, trunks[i].map);
174 
175 		kfree(trunks);
176 	}
177 
178 	kfree(buf);
179 }
180 
181 /*
182  * Allocate the dma buffer for storing ROCEE table entries
183  *
184  * @size: required size
185  * @page_shift: the unit size in a continuous dma address range
186  * @flags: HNS_ROCE_BUF_ flags to control the allocation flow.
187  */
188 struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size,
189 					u32 page_shift, u32 flags)
190 {
191 	u32 trunk_size, page_size, alloced_size;
192 	struct hns_roce_buf_list *trunks;
193 	struct hns_roce_buf *buf;
194 	gfp_t gfp_flags;
195 	u32 ntrunk, i;
196 
197 	/* The minimum shift of the page accessed by hw is HNS_HW_PAGE_SHIFT */
198 	if (WARN_ON(page_shift < HNS_HW_PAGE_SHIFT))
199 		return ERR_PTR(-EINVAL);
200 
201 	gfp_flags = (flags & HNS_ROCE_BUF_NOSLEEP) ? GFP_ATOMIC : GFP_KERNEL;
202 	buf = kzalloc(sizeof(*buf), gfp_flags);
203 	if (!buf)
204 		return ERR_PTR(-ENOMEM);
205 
206 	buf->page_shift = page_shift;
207 	page_size = 1 << buf->page_shift;
208 
209 	/* Calc the trunk size and num by required size and page_shift */
210 	if (flags & HNS_ROCE_BUF_DIRECT) {
211 		buf->trunk_shift = ilog2(ALIGN(size, PAGE_SIZE));
212 		ntrunk = 1;
213 	} else {
214 		buf->trunk_shift = ilog2(ALIGN(page_size, PAGE_SIZE));
215 		ntrunk = DIV_ROUND_UP(size, 1 << buf->trunk_shift);
216 	}
217 
218 	trunks = kcalloc(ntrunk, sizeof(*trunks), gfp_flags);
219 	if (!trunks) {
220 		kfree(buf);
221 		return ERR_PTR(-ENOMEM);
222 	}
223 
224 	trunk_size = 1 << buf->trunk_shift;
225 	alloced_size = 0;
226 	for (i = 0; i < ntrunk; i++) {
227 		trunks[i].buf = dma_alloc_coherent(hr_dev->dev, trunk_size,
228 						   &trunks[i].map, gfp_flags);
229 		if (!trunks[i].buf)
230 			break;
231 
232 		alloced_size += trunk_size;
233 	}
234 
235 	buf->ntrunks = i;
236 
237 	/* In nofail mode, it's only failed when the alloced size is 0 */
238 	if ((flags & HNS_ROCE_BUF_NOFAIL) ? i == 0 : i != ntrunk) {
239 		for (i = 0; i < buf->ntrunks; i++)
240 			dma_free_coherent(hr_dev->dev, trunk_size,
241 					  trunks[i].buf, trunks[i].map);
242 
243 		kfree(trunks);
244 		kfree(buf);
245 		return ERR_PTR(-ENOMEM);
246 	}
247 
248 	buf->npages = DIV_ROUND_UP(alloced_size, page_size);
249 	buf->trunk_list = trunks;
250 
251 	return buf;
252 }
253 
254 int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
255 			   int buf_cnt, int start, struct hns_roce_buf *buf)
256 {
257 	int i, end;
258 	int total;
259 
260 	end = start + buf_cnt;
261 	if (end > buf->npages) {
262 		dev_err(hr_dev->dev,
263 			"failed to check kmem bufs, end %d + %d total %u!\n",
264 			start, buf_cnt, buf->npages);
265 		return -EINVAL;
266 	}
267 
268 	total = 0;
269 	for (i = start; i < end; i++)
270 		bufs[total++] = hns_roce_buf_page(buf, i);
271 
272 	return total;
273 }
274 
275 int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
276 			   int buf_cnt, int start, struct ib_umem *umem,
277 			   unsigned int page_shift)
278 {
279 	struct ib_block_iter biter;
280 	int total = 0;
281 	int idx = 0;
282 	u64 addr;
283 
284 	if (page_shift < HNS_HW_PAGE_SHIFT) {
285 		dev_err(hr_dev->dev, "failed to check umem page shift %u!\n",
286 			page_shift);
287 		return -EINVAL;
288 	}
289 
290 	/* convert system page cnt to hw page cnt */
291 	rdma_umem_for_each_dma_block(umem, &biter, 1 << page_shift) {
292 		addr = rdma_block_iter_dma_address(&biter);
293 		if (idx >= start) {
294 			bufs[total++] = addr;
295 			if (total >= buf_cnt)
296 				goto done;
297 		}
298 		idx++;
299 	}
300 
301 done:
302 	return total;
303 }
304 
305 void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev)
306 {
307 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
308 		hns_roce_cleanup_srq_table(hr_dev);
309 	hns_roce_cleanup_qp_table(hr_dev);
310 	hns_roce_cleanup_cq_table(hr_dev);
311 	hns_roce_cleanup_mr_table(hr_dev);
312 	hns_roce_cleanup_pd_table(hr_dev);
313 	hns_roce_cleanup_uar_table(hr_dev);
314 }
315