1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * AMD Passthrough DMA device driver
4  * -- Based on the CCP driver
5  *
6  * Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
7  *
8  * Author: Sanjay R Mehta <sanju.mehta@amd.com>
9  * Author: Gary R Hook <gary.hook@amd.com>
10  */
11 
12 #include "ptdma.h"
13 #include "../dmaengine.h"
14 #include "../virt-dma.h"
15 
16 static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan)
17 {
18 	return container_of(dma_chan, struct pt_dma_chan, vc.chan);
19 }
20 
21 static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd)
22 {
23 	return container_of(vd, struct pt_dma_desc, vd);
24 }
25 
26 static void pt_free_chan_resources(struct dma_chan *dma_chan)
27 {
28 	struct pt_dma_chan *chan = to_pt_chan(dma_chan);
29 
30 	vchan_free_chan_resources(&chan->vc);
31 }
32 
33 static void pt_synchronize(struct dma_chan *dma_chan)
34 {
35 	struct pt_dma_chan *chan = to_pt_chan(dma_chan);
36 
37 	vchan_synchronize(&chan->vc);
38 }
39 
40 static void pt_do_cleanup(struct virt_dma_desc *vd)
41 {
42 	struct pt_dma_desc *desc = to_pt_desc(vd);
43 	struct pt_device *pt = desc->pt;
44 
45 	kmem_cache_free(pt->dma_desc_cache, desc);
46 }
47 
48 static int pt_dma_start_desc(struct pt_dma_desc *desc)
49 {
50 	struct pt_passthru_engine *pt_engine;
51 	struct pt_device *pt;
52 	struct pt_cmd *pt_cmd;
53 	struct pt_cmd_queue *cmd_q;
54 
55 	desc->issued_to_hw = 1;
56 
57 	pt_cmd = &desc->pt_cmd;
58 	pt = pt_cmd->pt;
59 	cmd_q = &pt->cmd_q;
60 	pt_engine = &pt_cmd->passthru;
61 
62 	pt->tdata.cmd = pt_cmd;
63 
64 	/* Execute the command */
65 	pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine);
66 
67 	return 0;
68 }
69 
70 static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan)
71 {
72 	/* Get the next DMA descriptor on the active list */
73 	struct virt_dma_desc *vd = vchan_next_desc(&chan->vc);
74 
75 	return vd ? to_pt_desc(vd) : NULL;
76 }
77 
78 static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan,
79 						 struct pt_dma_desc *desc)
80 {
81 	struct dma_async_tx_descriptor *tx_desc;
82 	struct virt_dma_desc *vd;
83 	unsigned long flags;
84 
85 	/* Loop over descriptors until one is found with commands */
86 	do {
87 		if (desc) {
88 			if (!desc->issued_to_hw) {
89 				/* No errors, keep going */
90 				if (desc->status != DMA_ERROR)
91 					return desc;
92 			}
93 
94 			tx_desc = &desc->vd.tx;
95 			vd = &desc->vd;
96 		} else {
97 			tx_desc = NULL;
98 		}
99 
100 		spin_lock_irqsave(&chan->vc.lock, flags);
101 
102 		if (desc) {
103 			if (desc->status != DMA_COMPLETE) {
104 				if (desc->status != DMA_ERROR)
105 					desc->status = DMA_COMPLETE;
106 
107 				dma_cookie_complete(tx_desc);
108 				dma_descriptor_unmap(tx_desc);
109 				list_del(&desc->vd.node);
110 			} else {
111 				/* Don't handle it twice */
112 				tx_desc = NULL;
113 			}
114 		}
115 
116 		desc = pt_next_dma_desc(chan);
117 
118 		spin_unlock_irqrestore(&chan->vc.lock, flags);
119 
120 		if (tx_desc) {
121 			dmaengine_desc_get_callback_invoke(tx_desc, NULL);
122 			dma_run_dependencies(tx_desc);
123 			vchan_vdesc_fini(vd);
124 		}
125 	} while (desc);
126 
127 	return NULL;
128 }
129 
130 static void pt_cmd_callback(void *data, int err)
131 {
132 	struct pt_dma_desc *desc = data;
133 	struct dma_chan *dma_chan;
134 	struct pt_dma_chan *chan;
135 	int ret;
136 
137 	if (err == -EINPROGRESS)
138 		return;
139 
140 	dma_chan = desc->vd.tx.chan;
141 	chan = to_pt_chan(dma_chan);
142 
143 	if (err)
144 		desc->status = DMA_ERROR;
145 
146 	while (true) {
147 		/* Check for DMA descriptor completion */
148 		desc = pt_handle_active_desc(chan, desc);
149 
150 		/* Don't submit cmd if no descriptor or DMA is paused */
151 		if (!desc)
152 			break;
153 
154 		ret = pt_dma_start_desc(desc);
155 		if (!ret)
156 			break;
157 
158 		desc->status = DMA_ERROR;
159 	}
160 }
161 
162 static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan,
163 					     unsigned long flags)
164 {
165 	struct pt_dma_desc *desc;
166 
167 	desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT);
168 	if (!desc)
169 		return NULL;
170 
171 	vchan_tx_prep(&chan->vc, &desc->vd, flags);
172 
173 	desc->pt = chan->pt;
174 	desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT);
175 	desc->issued_to_hw = 0;
176 	desc->status = DMA_IN_PROGRESS;
177 
178 	return desc;
179 }
180 
181 static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan,
182 					  dma_addr_t dst,
183 					  dma_addr_t src,
184 					  unsigned int len,
185 					  unsigned long flags)
186 {
187 	struct pt_dma_chan *chan = to_pt_chan(dma_chan);
188 	struct pt_passthru_engine *pt_engine;
189 	struct pt_dma_desc *desc;
190 	struct pt_cmd *pt_cmd;
191 
192 	desc = pt_alloc_dma_desc(chan, flags);
193 	if (!desc)
194 		return NULL;
195 
196 	pt_cmd = &desc->pt_cmd;
197 	pt_cmd->pt = chan->pt;
198 	pt_engine = &pt_cmd->passthru;
199 	pt_cmd->engine = PT_ENGINE_PASSTHRU;
200 	pt_engine->src_dma = src;
201 	pt_engine->dst_dma = dst;
202 	pt_engine->src_len = len;
203 	pt_cmd->pt_cmd_callback = pt_cmd_callback;
204 	pt_cmd->data = desc;
205 
206 	desc->len = len;
207 
208 	return desc;
209 }
210 
211 static struct dma_async_tx_descriptor *
212 pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst,
213 		   dma_addr_t src, size_t len, unsigned long flags)
214 {
215 	struct pt_dma_desc *desc;
216 
217 	desc = pt_create_desc(dma_chan, dst, src, len, flags);
218 	if (!desc)
219 		return NULL;
220 
221 	return &desc->vd.tx;
222 }
223 
224 static struct dma_async_tx_descriptor *
225 pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags)
226 {
227 	struct pt_dma_chan *chan = to_pt_chan(dma_chan);
228 	struct pt_dma_desc *desc;
229 
230 	desc = pt_alloc_dma_desc(chan, flags);
231 	if (!desc)
232 		return NULL;
233 
234 	return &desc->vd.tx;
235 }
236 
237 static void pt_issue_pending(struct dma_chan *dma_chan)
238 {
239 	struct pt_dma_chan *chan = to_pt_chan(dma_chan);
240 	struct pt_dma_desc *desc;
241 	unsigned long flags;
242 	bool engine_is_idle = true;
243 
244 	spin_lock_irqsave(&chan->vc.lock, flags);
245 
246 	desc = pt_next_dma_desc(chan);
247 	if (desc)
248 		engine_is_idle = false;
249 
250 	vchan_issue_pending(&chan->vc);
251 
252 	desc = pt_next_dma_desc(chan);
253 
254 	spin_unlock_irqrestore(&chan->vc.lock, flags);
255 
256 	/* If there was nothing active, start processing */
257 	if (engine_is_idle)
258 		pt_cmd_callback(desc, 0);
259 }
260 
261 static enum dma_status
262 pt_tx_status(struct dma_chan *c, dma_cookie_t cookie,
263 		struct dma_tx_state *txstate)
264 {
265 	struct pt_device *pt = to_pt_chan(c)->pt;
266 	struct pt_cmd_queue *cmd_q = &pt->cmd_q;
267 
268 	pt_check_status_trans(pt, cmd_q);
269 	return dma_cookie_status(c, cookie, txstate);
270 }
271 
272 static int pt_pause(struct dma_chan *dma_chan)
273 {
274 	struct pt_dma_chan *chan = to_pt_chan(dma_chan);
275 	unsigned long flags;
276 
277 	spin_lock_irqsave(&chan->vc.lock, flags);
278 	pt_stop_queue(&chan->pt->cmd_q);
279 	spin_unlock_irqrestore(&chan->vc.lock, flags);
280 
281 	return 0;
282 }
283 
284 static int pt_resume(struct dma_chan *dma_chan)
285 {
286 	struct pt_dma_chan *chan = to_pt_chan(dma_chan);
287 	struct pt_dma_desc *desc = NULL;
288 	unsigned long flags;
289 
290 	spin_lock_irqsave(&chan->vc.lock, flags);
291 	pt_start_queue(&chan->pt->cmd_q);
292 	desc = pt_next_dma_desc(chan);
293 	spin_unlock_irqrestore(&chan->vc.lock, flags);
294 
295 	/* If there was something active, re-start */
296 	if (desc)
297 		pt_cmd_callback(desc, 0);
298 
299 	return 0;
300 }
301 
302 static int pt_terminate_all(struct dma_chan *dma_chan)
303 {
304 	struct pt_dma_chan *chan = to_pt_chan(dma_chan);
305 	unsigned long flags;
306 	struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q;
307 	LIST_HEAD(head);
308 
309 	iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
310 	spin_lock_irqsave(&chan->vc.lock, flags);
311 	vchan_get_all_descriptors(&chan->vc, &head);
312 	spin_unlock_irqrestore(&chan->vc.lock, flags);
313 
314 	vchan_dma_desc_free_list(&chan->vc, &head);
315 	vchan_free_chan_resources(&chan->vc);
316 
317 	return 0;
318 }
319 
320 int pt_dmaengine_register(struct pt_device *pt)
321 {
322 	struct pt_dma_chan *chan;
323 	struct dma_device *dma_dev = &pt->dma_dev;
324 	char *cmd_cache_name;
325 	char *desc_cache_name;
326 	int ret;
327 
328 	pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan),
329 				       GFP_KERNEL);
330 	if (!pt->pt_dma_chan)
331 		return -ENOMEM;
332 
333 	cmd_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL,
334 					"%s-dmaengine-cmd-cache",
335 					dev_name(pt->dev));
336 	if (!cmd_cache_name)
337 		return -ENOMEM;
338 
339 	desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL,
340 					 "%s-dmaengine-desc-cache",
341 					 dev_name(pt->dev));
342 	if (!desc_cache_name) {
343 		ret = -ENOMEM;
344 		goto err_cache;
345 	}
346 
347 	pt->dma_desc_cache = kmem_cache_create(desc_cache_name,
348 					       sizeof(struct pt_dma_desc), 0,
349 					       SLAB_HWCACHE_ALIGN, NULL);
350 	if (!pt->dma_desc_cache) {
351 		ret = -ENOMEM;
352 		goto err_cache;
353 	}
354 
355 	dma_dev->dev = pt->dev;
356 	dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
357 	dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
358 	dma_dev->directions = DMA_MEM_TO_MEM;
359 	dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
360 	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
361 	dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
362 
363 	/*
364 	 * PTDMA is intended to be used with the AMD NTB devices, hence
365 	 * marking it as DMA_PRIVATE.
366 	 */
367 	dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
368 
369 	INIT_LIST_HEAD(&dma_dev->channels);
370 
371 	chan = pt->pt_dma_chan;
372 	chan->pt = pt;
373 
374 	/* Set base and prep routines */
375 	dma_dev->device_free_chan_resources = pt_free_chan_resources;
376 	dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy;
377 	dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt;
378 	dma_dev->device_issue_pending = pt_issue_pending;
379 	dma_dev->device_tx_status = pt_tx_status;
380 	dma_dev->device_pause = pt_pause;
381 	dma_dev->device_resume = pt_resume;
382 	dma_dev->device_terminate_all = pt_terminate_all;
383 	dma_dev->device_synchronize = pt_synchronize;
384 
385 	chan->vc.desc_free = pt_do_cleanup;
386 	vchan_init(&chan->vc, dma_dev);
387 
388 	dma_set_mask_and_coherent(pt->dev, DMA_BIT_MASK(64));
389 
390 	ret = dma_async_device_register(dma_dev);
391 	if (ret)
392 		goto err_reg;
393 
394 	return 0;
395 
396 err_reg:
397 	kmem_cache_destroy(pt->dma_desc_cache);
398 
399 err_cache:
400 	kmem_cache_destroy(pt->dma_cmd_cache);
401 
402 	return ret;
403 }
404 
405 void pt_dmaengine_unregister(struct pt_device *pt)
406 {
407 	struct dma_device *dma_dev = &pt->dma_dev;
408 
409 	dma_async_device_unregister(dma_dev);
410 
411 	kmem_cache_destroy(pt->dma_desc_cache);
412 	kmem_cache_destroy(pt->dma_cmd_cache);
413 }
414