1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 /* Copyright (c) 2020 Marvell International Ltd. */
3 
4 #include <linux/dma-mapping.h>
5 #include <linux/qed/qed_chain.h>
6 #include <linux/vmalloc.h>
7 
8 #include "qed_dev_api.h"
9 
10 static void qed_chain_init(struct qed_chain *chain,
11 			   const struct qed_chain_init_params *params,
12 			   u32 page_cnt)
13 {
14 	memset(chain, 0, sizeof(*chain));
15 
16 	chain->elem_size = params->elem_size;
17 	chain->intended_use = params->intended_use;
18 	chain->mode = params->mode;
19 	chain->cnt_type = params->cnt_type;
20 
21 	chain->elem_per_page = ELEMS_PER_PAGE(params->elem_size,
22 					      params->page_size);
23 	chain->usable_per_page = USABLE_ELEMS_PER_PAGE(params->elem_size,
24 						       params->page_size,
25 						       params->mode);
26 	chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(params->elem_size,
27 						       params->mode);
28 
29 	chain->elem_per_page_mask = chain->elem_per_page - 1;
30 	chain->next_page_mask = chain->usable_per_page &
31 				chain->elem_per_page_mask;
32 
33 	chain->page_size = params->page_size;
34 	chain->page_cnt = page_cnt;
35 	chain->capacity = chain->usable_per_page * page_cnt;
36 	chain->size = chain->elem_per_page * page_cnt;
37 
38 	if (params->ext_pbl_virt) {
39 		chain->pbl_sp.table_virt = params->ext_pbl_virt;
40 		chain->pbl_sp.table_phys = params->ext_pbl_phys;
41 
42 		chain->b_external_pbl = true;
43 	}
44 }
45 
46 static void qed_chain_init_next_ptr_elem(const struct qed_chain *chain,
47 					 void *virt_curr, void *virt_next,
48 					 dma_addr_t phys_next)
49 {
50 	struct qed_chain_next *next;
51 	u32 size;
52 
53 	size = chain->elem_size * chain->usable_per_page;
54 	next = virt_curr + size;
55 
56 	DMA_REGPAIR_LE(next->next_phys, phys_next);
57 	next->next_virt = virt_next;
58 }
59 
60 static void qed_chain_init_mem(struct qed_chain *chain, void *virt_addr,
61 			       dma_addr_t phys_addr)
62 {
63 	chain->p_virt_addr = virt_addr;
64 	chain->p_phys_addr = phys_addr;
65 }
66 
67 static void qed_chain_free_next_ptr(struct qed_dev *cdev,
68 				    struct qed_chain *chain)
69 {
70 	struct device *dev = &cdev->pdev->dev;
71 	struct qed_chain_next *next;
72 	dma_addr_t phys, phys_next;
73 	void *virt, *virt_next;
74 	u32 size, i;
75 
76 	size = chain->elem_size * chain->usable_per_page;
77 	virt = chain->p_virt_addr;
78 	phys = chain->p_phys_addr;
79 
80 	for (i = 0; i < chain->page_cnt; i++) {
81 		if (!virt)
82 			break;
83 
84 		next = virt + size;
85 		virt_next = next->next_virt;
86 		phys_next = HILO_DMA_REGPAIR(next->next_phys);
87 
88 		dma_free_coherent(dev, chain->page_size, virt, phys);
89 
90 		virt = virt_next;
91 		phys = phys_next;
92 	}
93 }
94 
95 static void qed_chain_free_single(struct qed_dev *cdev,
96 				  struct qed_chain *chain)
97 {
98 	if (!chain->p_virt_addr)
99 		return;
100 
101 	dma_free_coherent(&cdev->pdev->dev, chain->page_size,
102 			  chain->p_virt_addr, chain->p_phys_addr);
103 }
104 
105 static void qed_chain_free_pbl(struct qed_dev *cdev, struct qed_chain *chain)
106 {
107 	struct device *dev = &cdev->pdev->dev;
108 	struct addr_tbl_entry *entry;
109 	u32 i;
110 
111 	if (!chain->pbl.pp_addr_tbl)
112 		return;
113 
114 	for (i = 0; i < chain->page_cnt; i++) {
115 		entry = chain->pbl.pp_addr_tbl + i;
116 		if (!entry->virt_addr)
117 			break;
118 
119 		dma_free_coherent(dev, chain->page_size, entry->virt_addr,
120 				  entry->dma_map);
121 	}
122 
123 	if (!chain->b_external_pbl)
124 		dma_free_coherent(dev, chain->pbl_sp.table_size,
125 				  chain->pbl_sp.table_virt,
126 				  chain->pbl_sp.table_phys);
127 
128 	vfree(chain->pbl.pp_addr_tbl);
129 	chain->pbl.pp_addr_tbl = NULL;
130 }
131 
132 /**
133  * qed_chain_free() - Free chain DMA memory.
134  *
135  * @cdev: Main device structure.
136  * @chain: Chain to free.
137  */
138 void qed_chain_free(struct qed_dev *cdev, struct qed_chain *chain)
139 {
140 	switch (chain->mode) {
141 	case QED_CHAIN_MODE_NEXT_PTR:
142 		qed_chain_free_next_ptr(cdev, chain);
143 		break;
144 	case QED_CHAIN_MODE_SINGLE:
145 		qed_chain_free_single(cdev, chain);
146 		break;
147 	case QED_CHAIN_MODE_PBL:
148 		qed_chain_free_pbl(cdev, chain);
149 		break;
150 	default:
151 		return;
152 	}
153 
154 	qed_chain_init_mem(chain, NULL, 0);
155 }
156 
157 static int
158 qed_chain_alloc_sanity_check(struct qed_dev *cdev,
159 			     const struct qed_chain_init_params *params,
160 			     u32 page_cnt)
161 {
162 	u64 chain_size;
163 
164 	chain_size = ELEMS_PER_PAGE(params->elem_size, params->page_size);
165 	chain_size *= page_cnt;
166 
167 	if (!chain_size)
168 		return -EINVAL;
169 
170 	/* The actual chain size can be larger than the maximal possible value
171 	 * after rounding up the requested elements number to pages, and after
172 	 * taking into account the unusuable elements (next-ptr elements).
173 	 * The size of a "u16" chain can be (U16_MAX + 1) since the chain
174 	 * size/capacity fields are of u32 type.
175 	 */
176 	switch (params->cnt_type) {
177 	case QED_CHAIN_CNT_TYPE_U16:
178 		if (chain_size > U16_MAX + 1)
179 			break;
180 
181 		return 0;
182 	case QED_CHAIN_CNT_TYPE_U32:
183 		if (chain_size > U32_MAX)
184 			break;
185 
186 		return 0;
187 	default:
188 		return -EINVAL;
189 	}
190 
191 	DP_NOTICE(cdev,
192 		  "The actual chain size (0x%llx) is larger than the maximal possible value\n",
193 		  chain_size);
194 
195 	return -EINVAL;
196 }
197 
198 static int qed_chain_alloc_next_ptr(struct qed_dev *cdev,
199 				    struct qed_chain *chain)
200 {
201 	struct device *dev = &cdev->pdev->dev;
202 	void *virt, *virt_prev = NULL;
203 	dma_addr_t phys;
204 	u32 i;
205 
206 	for (i = 0; i < chain->page_cnt; i++) {
207 		virt = dma_alloc_coherent(dev, chain->page_size, &phys,
208 					  GFP_KERNEL);
209 		if (!virt)
210 			return -ENOMEM;
211 
212 		if (i == 0) {
213 			qed_chain_init_mem(chain, virt, phys);
214 			qed_chain_reset(chain);
215 		} else {
216 			qed_chain_init_next_ptr_elem(chain, virt_prev, virt,
217 						     phys);
218 		}
219 
220 		virt_prev = virt;
221 	}
222 
223 	/* Last page's next element should point to the beginning of the
224 	 * chain.
225 	 */
226 	qed_chain_init_next_ptr_elem(chain, virt_prev, chain->p_virt_addr,
227 				     chain->p_phys_addr);
228 
229 	return 0;
230 }
231 
232 static int qed_chain_alloc_single(struct qed_dev *cdev,
233 				  struct qed_chain *chain)
234 {
235 	dma_addr_t phys;
236 	void *virt;
237 
238 	virt = dma_alloc_coherent(&cdev->pdev->dev, chain->page_size,
239 				  &phys, GFP_KERNEL);
240 	if (!virt)
241 		return -ENOMEM;
242 
243 	qed_chain_init_mem(chain, virt, phys);
244 	qed_chain_reset(chain);
245 
246 	return 0;
247 }
248 
249 static int qed_chain_alloc_pbl(struct qed_dev *cdev, struct qed_chain *chain)
250 {
251 	struct device *dev = &cdev->pdev->dev;
252 	struct addr_tbl_entry *addr_tbl;
253 	dma_addr_t phys, pbl_phys;
254 	__le64 *pbl_virt;
255 	u32 page_cnt, i;
256 	size_t size;
257 	void *virt;
258 
259 	page_cnt = chain->page_cnt;
260 
261 	size = array_size(page_cnt, sizeof(*addr_tbl));
262 	if (unlikely(size == SIZE_MAX))
263 		return -EOVERFLOW;
264 
265 	addr_tbl = vzalloc(size);
266 	if (!addr_tbl)
267 		return -ENOMEM;
268 
269 	chain->pbl.pp_addr_tbl = addr_tbl;
270 
271 	if (chain->b_external_pbl) {
272 		pbl_virt = chain->pbl_sp.table_virt;
273 		goto alloc_pages;
274 	}
275 
276 	size = array_size(page_cnt, sizeof(*pbl_virt));
277 	if (unlikely(size == SIZE_MAX))
278 		return -EOVERFLOW;
279 
280 	pbl_virt = dma_alloc_coherent(dev, size, &pbl_phys, GFP_KERNEL);
281 	if (!pbl_virt)
282 		return -ENOMEM;
283 
284 	chain->pbl_sp.table_virt = pbl_virt;
285 	chain->pbl_sp.table_phys = pbl_phys;
286 	chain->pbl_sp.table_size = size;
287 
288 alloc_pages:
289 	for (i = 0; i < page_cnt; i++) {
290 		virt = dma_alloc_coherent(dev, chain->page_size, &phys,
291 					  GFP_KERNEL);
292 		if (!virt)
293 			return -ENOMEM;
294 
295 		if (i == 0) {
296 			qed_chain_init_mem(chain, virt, phys);
297 			qed_chain_reset(chain);
298 		}
299 
300 		/* Fill the PBL table with the physical address of the page */
301 		pbl_virt[i] = cpu_to_le64(phys);
302 
303 		/* Keep the virtual address of the page */
304 		addr_tbl[i].virt_addr = virt;
305 		addr_tbl[i].dma_map = phys;
306 	}
307 
308 	return 0;
309 }
310 
311 /**
312  * qed_chain_alloc() - Allocate and initialize a chain.
313  *
314  * @cdev: Main device structure.
315  * @chain: Chain to be processed.
316  * @params: Chain initialization parameters.
317  *
318  * Return: 0 on success, negative errno otherwise.
319  */
320 int qed_chain_alloc(struct qed_dev *cdev, struct qed_chain *chain,
321 		    struct qed_chain_init_params *params)
322 {
323 	u32 page_cnt;
324 	int rc;
325 
326 	if (!params->page_size)
327 		params->page_size = QED_CHAIN_PAGE_SIZE;
328 
329 	if (params->mode == QED_CHAIN_MODE_SINGLE)
330 		page_cnt = 1;
331 	else
332 		page_cnt = QED_CHAIN_PAGE_CNT(params->num_elems,
333 					      params->elem_size,
334 					      params->page_size,
335 					      params->mode);
336 
337 	rc = qed_chain_alloc_sanity_check(cdev, params, page_cnt);
338 	if (rc) {
339 		DP_NOTICE(cdev,
340 			  "Cannot allocate a chain with the given arguments:\n");
341 		DP_NOTICE(cdev,
342 			  "[use_mode %d, mode %d, cnt_type %d, num_elems %d, elem_size %zu, page_size %u]\n",
343 			  params->intended_use, params->mode, params->cnt_type,
344 			  params->num_elems, params->elem_size,
345 			  params->page_size);
346 		return rc;
347 	}
348 
349 	qed_chain_init(chain, params, page_cnt);
350 
351 	switch (params->mode) {
352 	case QED_CHAIN_MODE_NEXT_PTR:
353 		rc = qed_chain_alloc_next_ptr(cdev, chain);
354 		break;
355 	case QED_CHAIN_MODE_SINGLE:
356 		rc = qed_chain_alloc_single(cdev, chain);
357 		break;
358 	case QED_CHAIN_MODE_PBL:
359 		rc = qed_chain_alloc_pbl(cdev, chain);
360 		break;
361 	default:
362 		return -EINVAL;
363 	}
364 
365 	if (!rc)
366 		return 0;
367 
368 	qed_chain_free(cdev, chain);
369 
370 	return rc;
371 }
372