xref: /openbmc/linux/drivers/dma/dmatest.c (revision 2f828fb2)
1 /*
2  * DMA Engine test module
3  *
4  * Copyright (C) 2007 Atmel Corporation
5  * Copyright (C) 2013 Intel Corporation
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 
13 #include <linux/delay.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dmaengine.h>
16 #include <linux/freezer.h>
17 #include <linux/init.h>
18 #include <linux/kthread.h>
19 #include <linux/sched/task.h>
20 #include <linux/module.h>
21 #include <linux/moduleparam.h>
22 #include <linux/random.h>
23 #include <linux/slab.h>
24 #include <linux/wait.h>
25 
26 static unsigned int test_buf_size = 16384;
27 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
28 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
29 
30 static char test_channel[20];
31 module_param_string(channel, test_channel, sizeof(test_channel),
32 		S_IRUGO | S_IWUSR);
33 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
34 
35 static char test_device[32];
36 module_param_string(device, test_device, sizeof(test_device),
37 		S_IRUGO | S_IWUSR);
38 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
39 
40 static unsigned int threads_per_chan = 1;
41 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
42 MODULE_PARM_DESC(threads_per_chan,
43 		"Number of threads to start per channel (default: 1)");
44 
45 static unsigned int max_channels;
46 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
47 MODULE_PARM_DESC(max_channels,
48 		"Maximum number of channels to use (default: all)");
49 
50 static unsigned int iterations;
51 module_param(iterations, uint, S_IRUGO | S_IWUSR);
52 MODULE_PARM_DESC(iterations,
53 		"Iterations before stopping test (default: infinite)");
54 
55 static unsigned int dmatest;
56 module_param(dmatest, uint, S_IRUGO | S_IWUSR);
57 MODULE_PARM_DESC(dmatest,
58 		"dmatest 0-memcpy 1-memset (default: 0)");
59 
60 static unsigned int xor_sources = 3;
61 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
62 MODULE_PARM_DESC(xor_sources,
63 		"Number of xor source buffers (default: 3)");
64 
65 static unsigned int pq_sources = 3;
66 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
67 MODULE_PARM_DESC(pq_sources,
68 		"Number of p+q source buffers (default: 3)");
69 
70 static int timeout = 3000;
71 module_param(timeout, uint, S_IRUGO | S_IWUSR);
72 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
73 		 "Pass -1 for infinite timeout");
74 
75 static bool noverify;
76 module_param(noverify, bool, S_IRUGO | S_IWUSR);
77 MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
78 
79 static bool verbose;
80 module_param(verbose, bool, S_IRUGO | S_IWUSR);
81 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
82 
83 /**
84  * struct dmatest_params - test parameters.
85  * @buf_size:		size of the memcpy test buffer
86  * @channel:		bus ID of the channel to test
87  * @device:		bus ID of the DMA Engine to test
88  * @threads_per_chan:	number of threads to start per channel
89  * @max_channels:	maximum number of channels to use
90  * @iterations:		iterations before stopping test
91  * @xor_sources:	number of xor source buffers
92  * @pq_sources:		number of p+q source buffers
93  * @timeout:		transfer timeout in msec, -1 for infinite timeout
94  */
95 struct dmatest_params {
96 	unsigned int	buf_size;
97 	char		channel[20];
98 	char		device[32];
99 	unsigned int	threads_per_chan;
100 	unsigned int	max_channels;
101 	unsigned int	iterations;
102 	unsigned int	xor_sources;
103 	unsigned int	pq_sources;
104 	int		timeout;
105 	bool		noverify;
106 };
107 
108 /**
109  * struct dmatest_info - test information.
110  * @params:		test parameters
111  * @lock:		access protection to the fields of this structure
112  */
113 static struct dmatest_info {
114 	/* Test parameters */
115 	struct dmatest_params	params;
116 
117 	/* Internal state */
118 	struct list_head	channels;
119 	unsigned int		nr_channels;
120 	struct mutex		lock;
121 	bool			did_init;
122 } test_info = {
123 	.channels = LIST_HEAD_INIT(test_info.channels),
124 	.lock = __MUTEX_INITIALIZER(test_info.lock),
125 };
126 
127 static int dmatest_run_set(const char *val, const struct kernel_param *kp);
128 static int dmatest_run_get(char *val, const struct kernel_param *kp);
129 static const struct kernel_param_ops run_ops = {
130 	.set = dmatest_run_set,
131 	.get = dmatest_run_get,
132 };
133 static bool dmatest_run;
134 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
135 MODULE_PARM_DESC(run, "Run the test (default: false)");
136 
137 /* Maximum amount of mismatched bytes in buffer to print */
138 #define MAX_ERROR_COUNT		32
139 
140 /*
141  * Initialization patterns. All bytes in the source buffer has bit 7
142  * set, all bytes in the destination buffer has bit 7 cleared.
143  *
144  * Bit 6 is set for all bytes which are to be copied by the DMA
145  * engine. Bit 5 is set for all bytes which are to be overwritten by
146  * the DMA engine.
147  *
148  * The remaining bits are the inverse of a counter which increments by
149  * one for each byte address.
150  */
151 #define PATTERN_SRC		0x80
152 #define PATTERN_DST		0x00
153 #define PATTERN_COPY		0x40
154 #define PATTERN_OVERWRITE	0x20
155 #define PATTERN_COUNT_MASK	0x1f
156 #define PATTERN_MEMSET_IDX	0x01
157 
158 struct dmatest_thread {
159 	struct list_head	node;
160 	struct dmatest_info	*info;
161 	struct task_struct	*task;
162 	struct dma_chan		*chan;
163 	u8			**srcs;
164 	u8			**usrcs;
165 	u8			**dsts;
166 	u8			**udsts;
167 	enum dma_transaction_type type;
168 	bool			done;
169 };
170 
171 struct dmatest_chan {
172 	struct list_head	node;
173 	struct dma_chan		*chan;
174 	struct list_head	threads;
175 };
176 
177 static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
178 static bool wait;
179 
180 static bool is_threaded_test_run(struct dmatest_info *info)
181 {
182 	struct dmatest_chan *dtc;
183 
184 	list_for_each_entry(dtc, &info->channels, node) {
185 		struct dmatest_thread *thread;
186 
187 		list_for_each_entry(thread, &dtc->threads, node) {
188 			if (!thread->done)
189 				return true;
190 		}
191 	}
192 
193 	return false;
194 }
195 
196 static int dmatest_wait_get(char *val, const struct kernel_param *kp)
197 {
198 	struct dmatest_info *info = &test_info;
199 	struct dmatest_params *params = &info->params;
200 
201 	if (params->iterations)
202 		wait_event(thread_wait, !is_threaded_test_run(info));
203 	wait = true;
204 	return param_get_bool(val, kp);
205 }
206 
207 static const struct kernel_param_ops wait_ops = {
208 	.get = dmatest_wait_get,
209 	.set = param_set_bool,
210 };
211 module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
212 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
213 
214 static bool dmatest_match_channel(struct dmatest_params *params,
215 		struct dma_chan *chan)
216 {
217 	if (params->channel[0] == '\0')
218 		return true;
219 	return strcmp(dma_chan_name(chan), params->channel) == 0;
220 }
221 
222 static bool dmatest_match_device(struct dmatest_params *params,
223 		struct dma_device *device)
224 {
225 	if (params->device[0] == '\0')
226 		return true;
227 	return strcmp(dev_name(device->dev), params->device) == 0;
228 }
229 
230 static unsigned long dmatest_random(void)
231 {
232 	unsigned long buf;
233 
234 	prandom_bytes(&buf, sizeof(buf));
235 	return buf;
236 }
237 
238 static inline u8 gen_inv_idx(u8 index, bool is_memset)
239 {
240 	u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
241 
242 	return ~val & PATTERN_COUNT_MASK;
243 }
244 
245 static inline u8 gen_src_value(u8 index, bool is_memset)
246 {
247 	return PATTERN_SRC | gen_inv_idx(index, is_memset);
248 }
249 
250 static inline u8 gen_dst_value(u8 index, bool is_memset)
251 {
252 	return PATTERN_DST | gen_inv_idx(index, is_memset);
253 }
254 
255 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
256 		unsigned int buf_size, bool is_memset)
257 {
258 	unsigned int i;
259 	u8 *buf;
260 
261 	for (; (buf = *bufs); bufs++) {
262 		for (i = 0; i < start; i++)
263 			buf[i] = gen_src_value(i, is_memset);
264 		for ( ; i < start + len; i++)
265 			buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
266 		for ( ; i < buf_size; i++)
267 			buf[i] = gen_src_value(i, is_memset);
268 		buf++;
269 	}
270 }
271 
272 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
273 		unsigned int buf_size, bool is_memset)
274 {
275 	unsigned int i;
276 	u8 *buf;
277 
278 	for (; (buf = *bufs); bufs++) {
279 		for (i = 0; i < start; i++)
280 			buf[i] = gen_dst_value(i, is_memset);
281 		for ( ; i < start + len; i++)
282 			buf[i] = gen_dst_value(i, is_memset) |
283 						PATTERN_OVERWRITE;
284 		for ( ; i < buf_size; i++)
285 			buf[i] = gen_dst_value(i, is_memset);
286 	}
287 }
288 
289 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
290 		unsigned int counter, bool is_srcbuf, bool is_memset)
291 {
292 	u8		diff = actual ^ pattern;
293 	u8		expected = pattern | gen_inv_idx(counter, is_memset);
294 	const char	*thread_name = current->comm;
295 
296 	if (is_srcbuf)
297 		pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
298 			thread_name, index, expected, actual);
299 	else if ((pattern & PATTERN_COPY)
300 			&& (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
301 		pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
302 			thread_name, index, expected, actual);
303 	else if (diff & PATTERN_SRC)
304 		pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
305 			thread_name, index, expected, actual);
306 	else
307 		pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
308 			thread_name, index, expected, actual);
309 }
310 
311 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
312 		unsigned int end, unsigned int counter, u8 pattern,
313 		bool is_srcbuf, bool is_memset)
314 {
315 	unsigned int i;
316 	unsigned int error_count = 0;
317 	u8 actual;
318 	u8 expected;
319 	u8 *buf;
320 	unsigned int counter_orig = counter;
321 
322 	for (; (buf = *bufs); bufs++) {
323 		counter = counter_orig;
324 		for (i = start; i < end; i++) {
325 			actual = buf[i];
326 			expected = pattern | gen_inv_idx(counter, is_memset);
327 			if (actual != expected) {
328 				if (error_count < MAX_ERROR_COUNT)
329 					dmatest_mismatch(actual, pattern, i,
330 							 counter, is_srcbuf,
331 							 is_memset);
332 				error_count++;
333 			}
334 			counter++;
335 		}
336 	}
337 
338 	if (error_count > MAX_ERROR_COUNT)
339 		pr_warn("%s: %u errors suppressed\n",
340 			current->comm, error_count - MAX_ERROR_COUNT);
341 
342 	return error_count;
343 }
344 
345 /* poor man's completion - we want to use wait_event_freezable() on it */
346 struct dmatest_done {
347 	bool			done;
348 	wait_queue_head_t	*wait;
349 };
350 
351 static void dmatest_callback(void *arg)
352 {
353 	struct dmatest_done *done = arg;
354 
355 	done->done = true;
356 	wake_up_all(done->wait);
357 }
358 
359 static unsigned int min_odd(unsigned int x, unsigned int y)
360 {
361 	unsigned int val = min(x, y);
362 
363 	return val % 2 ? val : val - 1;
364 }
365 
366 static void result(const char *err, unsigned int n, unsigned int src_off,
367 		   unsigned int dst_off, unsigned int len, unsigned long data)
368 {
369 	pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
370 		current->comm, n, err, src_off, dst_off, len, data);
371 }
372 
373 static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
374 		       unsigned int dst_off, unsigned int len,
375 		       unsigned long data)
376 {
377 	pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
378 		 current->comm, n, err, src_off, dst_off, len, data);
379 }
380 
381 #define verbose_result(err, n, src_off, dst_off, len, data) ({	\
382 	if (verbose)						\
383 		result(err, n, src_off, dst_off, len, data);	\
384 	else							\
385 		dbg_result(err, n, src_off, dst_off, len, data);\
386 })
387 
388 static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
389 {
390 	unsigned long long per_sec = 1000000;
391 
392 	if (runtime <= 0)
393 		return 0;
394 
395 	/* drop precision until runtime is 32-bits */
396 	while (runtime > UINT_MAX) {
397 		runtime >>= 1;
398 		per_sec <<= 1;
399 	}
400 
401 	per_sec *= val;
402 	do_div(per_sec, runtime);
403 	return per_sec;
404 }
405 
406 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
407 {
408 	return dmatest_persec(runtime, len >> 10);
409 }
410 
411 /*
412  * This function repeatedly tests DMA transfers of various lengths and
413  * offsets for a given operation type until it is told to exit by
414  * kthread_stop(). There may be multiple threads running this function
415  * in parallel for a single channel, and there may be multiple channels
416  * being tested in parallel.
417  *
418  * Before each test, the source and destination buffer is initialized
419  * with a known pattern. This pattern is different depending on
420  * whether it's in an area which is supposed to be copied or
421  * overwritten, and different in the source and destination buffers.
422  * So if the DMA engine doesn't copy exactly what we tell it to copy,
423  * we'll notice.
424  */
425 static int dmatest_func(void *data)
426 {
427 	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
428 	struct dmatest_thread	*thread = data;
429 	struct dmatest_done	done = { .wait = &done_wait };
430 	struct dmatest_info	*info;
431 	struct dmatest_params	*params;
432 	struct dma_chan		*chan;
433 	struct dma_device	*dev;
434 	unsigned int		error_count;
435 	unsigned int		failed_tests = 0;
436 	unsigned int		total_tests = 0;
437 	dma_cookie_t		cookie;
438 	enum dma_status		status;
439 	enum dma_ctrl_flags 	flags;
440 	u8			*pq_coefs = NULL;
441 	int			ret;
442 	int			src_cnt;
443 	int			dst_cnt;
444 	int			i;
445 	ktime_t			ktime, start, diff;
446 	ktime_t			filltime = 0;
447 	ktime_t			comparetime = 0;
448 	s64			runtime = 0;
449 	unsigned long long	total_len = 0;
450 	u8			align = 0;
451 	bool			is_memset = false;
452 
453 	set_freezable();
454 
455 	ret = -ENOMEM;
456 
457 	smp_rmb();
458 	info = thread->info;
459 	params = &info->params;
460 	chan = thread->chan;
461 	dev = chan->device;
462 	if (thread->type == DMA_MEMCPY) {
463 		align = dev->copy_align;
464 		src_cnt = dst_cnt = 1;
465 	} else if (thread->type == DMA_MEMSET) {
466 		align = dev->fill_align;
467 		src_cnt = dst_cnt = 1;
468 		is_memset = true;
469 	} else if (thread->type == DMA_XOR) {
470 		/* force odd to ensure dst = src */
471 		src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
472 		dst_cnt = 1;
473 		align = dev->xor_align;
474 	} else if (thread->type == DMA_PQ) {
475 		/* force odd to ensure dst = src */
476 		src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
477 		dst_cnt = 2;
478 		align = dev->pq_align;
479 
480 		pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
481 		if (!pq_coefs)
482 			goto err_thread_type;
483 
484 		for (i = 0; i < src_cnt; i++)
485 			pq_coefs[i] = 1;
486 	} else
487 		goto err_thread_type;
488 
489 	thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
490 	if (!thread->srcs)
491 		goto err_srcs;
492 
493 	thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
494 	if (!thread->usrcs)
495 		goto err_usrcs;
496 
497 	for (i = 0; i < src_cnt; i++) {
498 		thread->usrcs[i] = kmalloc(params->buf_size + align,
499 					   GFP_KERNEL);
500 		if (!thread->usrcs[i])
501 			goto err_srcbuf;
502 
503 		/* align srcs to alignment restriction */
504 		if (align)
505 			thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align);
506 		else
507 			thread->srcs[i] = thread->usrcs[i];
508 	}
509 	thread->srcs[i] = NULL;
510 
511 	thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
512 	if (!thread->dsts)
513 		goto err_dsts;
514 
515 	thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
516 	if (!thread->udsts)
517 		goto err_udsts;
518 
519 	for (i = 0; i < dst_cnt; i++) {
520 		thread->udsts[i] = kmalloc(params->buf_size + align,
521 					   GFP_KERNEL);
522 		if (!thread->udsts[i])
523 			goto err_dstbuf;
524 
525 		/* align dsts to alignment restriction */
526 		if (align)
527 			thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align);
528 		else
529 			thread->dsts[i] = thread->udsts[i];
530 	}
531 	thread->dsts[i] = NULL;
532 
533 	set_user_nice(current, 10);
534 
535 	/*
536 	 * src and dst buffers are freed by ourselves below
537 	 */
538 	flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
539 
540 	ktime = ktime_get();
541 	while (!kthread_should_stop()
542 	       && !(params->iterations && total_tests >= params->iterations)) {
543 		struct dma_async_tx_descriptor *tx = NULL;
544 		struct dmaengine_unmap_data *um;
545 		dma_addr_t srcs[src_cnt];
546 		dma_addr_t *dsts;
547 		unsigned int src_off, dst_off, len;
548 
549 		total_tests++;
550 
551 		/* Check if buffer count fits into map count variable (u8) */
552 		if ((src_cnt + dst_cnt) >= 255) {
553 			pr_err("too many buffers (%d of 255 supported)\n",
554 			       src_cnt + dst_cnt);
555 			break;
556 		}
557 
558 		if (1 << align > params->buf_size) {
559 			pr_err("%u-byte buffer too small for %d-byte alignment\n",
560 			       params->buf_size, 1 << align);
561 			break;
562 		}
563 
564 		if (params->noverify)
565 			len = params->buf_size;
566 		else
567 			len = dmatest_random() % params->buf_size + 1;
568 
569 		len = (len >> align) << align;
570 		if (!len)
571 			len = 1 << align;
572 
573 		total_len += len;
574 
575 		if (params->noverify) {
576 			src_off = 0;
577 			dst_off = 0;
578 		} else {
579 			start = ktime_get();
580 			src_off = dmatest_random() % (params->buf_size - len + 1);
581 			dst_off = dmatest_random() % (params->buf_size - len + 1);
582 
583 			src_off = (src_off >> align) << align;
584 			dst_off = (dst_off >> align) << align;
585 
586 			dmatest_init_srcs(thread->srcs, src_off, len,
587 					  params->buf_size, is_memset);
588 			dmatest_init_dsts(thread->dsts, dst_off, len,
589 					  params->buf_size, is_memset);
590 
591 			diff = ktime_sub(ktime_get(), start);
592 			filltime = ktime_add(filltime, diff);
593 		}
594 
595 		um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt,
596 					      GFP_KERNEL);
597 		if (!um) {
598 			failed_tests++;
599 			result("unmap data NULL", total_tests,
600 			       src_off, dst_off, len, ret);
601 			continue;
602 		}
603 
604 		um->len = params->buf_size;
605 		for (i = 0; i < src_cnt; i++) {
606 			void *buf = thread->srcs[i];
607 			struct page *pg = virt_to_page(buf);
608 			unsigned long pg_off = offset_in_page(buf);
609 
610 			um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
611 						   um->len, DMA_TO_DEVICE);
612 			srcs[i] = um->addr[i] + src_off;
613 			ret = dma_mapping_error(dev->dev, um->addr[i]);
614 			if (ret) {
615 				dmaengine_unmap_put(um);
616 				result("src mapping error", total_tests,
617 				       src_off, dst_off, len, ret);
618 				failed_tests++;
619 				continue;
620 			}
621 			um->to_cnt++;
622 		}
623 		/* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
624 		dsts = &um->addr[src_cnt];
625 		for (i = 0; i < dst_cnt; i++) {
626 			void *buf = thread->dsts[i];
627 			struct page *pg = virt_to_page(buf);
628 			unsigned long pg_off = offset_in_page(buf);
629 
630 			dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
631 					       DMA_BIDIRECTIONAL);
632 			ret = dma_mapping_error(dev->dev, dsts[i]);
633 			if (ret) {
634 				dmaengine_unmap_put(um);
635 				result("dst mapping error", total_tests,
636 				       src_off, dst_off, len, ret);
637 				failed_tests++;
638 				continue;
639 			}
640 			um->bidi_cnt++;
641 		}
642 
643 		if (thread->type == DMA_MEMCPY)
644 			tx = dev->device_prep_dma_memcpy(chan,
645 							 dsts[0] + dst_off,
646 							 srcs[0], len, flags);
647 		else if (thread->type == DMA_MEMSET)
648 			tx = dev->device_prep_dma_memset(chan,
649 						dsts[0] + dst_off,
650 						*(thread->srcs[0] + src_off),
651 						len, flags);
652 		else if (thread->type == DMA_XOR)
653 			tx = dev->device_prep_dma_xor(chan,
654 						      dsts[0] + dst_off,
655 						      srcs, src_cnt,
656 						      len, flags);
657 		else if (thread->type == DMA_PQ) {
658 			dma_addr_t dma_pq[dst_cnt];
659 
660 			for (i = 0; i < dst_cnt; i++)
661 				dma_pq[i] = dsts[i] + dst_off;
662 			tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
663 						     src_cnt, pq_coefs,
664 						     len, flags);
665 		}
666 
667 		if (!tx) {
668 			dmaengine_unmap_put(um);
669 			result("prep error", total_tests, src_off,
670 			       dst_off, len, ret);
671 			msleep(100);
672 			failed_tests++;
673 			continue;
674 		}
675 
676 		done.done = false;
677 		tx->callback = dmatest_callback;
678 		tx->callback_param = &done;
679 		cookie = tx->tx_submit(tx);
680 
681 		if (dma_submit_error(cookie)) {
682 			dmaengine_unmap_put(um);
683 			result("submit error", total_tests, src_off,
684 			       dst_off, len, ret);
685 			msleep(100);
686 			failed_tests++;
687 			continue;
688 		}
689 		dma_async_issue_pending(chan);
690 
691 		wait_event_freezable_timeout(done_wait, done.done,
692 					     msecs_to_jiffies(params->timeout));
693 
694 		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
695 
696 		if (!done.done) {
697 			/*
698 			 * We're leaving the timed out dma operation with
699 			 * dangling pointer to done_wait.  To make this
700 			 * correct, we'll need to allocate wait_done for
701 			 * each test iteration and perform "who's gonna
702 			 * free it this time?" dancing.  For now, just
703 			 * leave it dangling.
704 			 */
705 			WARN(1, "dmatest: Kernel stack may be corrupted!!\n");
706 			dmaengine_unmap_put(um);
707 			result("test timed out", total_tests, src_off, dst_off,
708 			       len, 0);
709 			failed_tests++;
710 			continue;
711 		} else if (status != DMA_COMPLETE) {
712 			dmaengine_unmap_put(um);
713 			result(status == DMA_ERROR ?
714 			       "completion error status" :
715 			       "completion busy status", total_tests, src_off,
716 			       dst_off, len, ret);
717 			failed_tests++;
718 			continue;
719 		}
720 
721 		dmaengine_unmap_put(um);
722 
723 		if (params->noverify) {
724 			verbose_result("test passed", total_tests, src_off,
725 				       dst_off, len, 0);
726 			continue;
727 		}
728 
729 		start = ktime_get();
730 		pr_debug("%s: verifying source buffer...\n", current->comm);
731 		error_count = dmatest_verify(thread->srcs, 0, src_off,
732 				0, PATTERN_SRC, true, is_memset);
733 		error_count += dmatest_verify(thread->srcs, src_off,
734 				src_off + len, src_off,
735 				PATTERN_SRC | PATTERN_COPY, true, is_memset);
736 		error_count += dmatest_verify(thread->srcs, src_off + len,
737 				params->buf_size, src_off + len,
738 				PATTERN_SRC, true, is_memset);
739 
740 		pr_debug("%s: verifying dest buffer...\n", current->comm);
741 		error_count += dmatest_verify(thread->dsts, 0, dst_off,
742 				0, PATTERN_DST, false, is_memset);
743 
744 		error_count += dmatest_verify(thread->dsts, dst_off,
745 				dst_off + len, src_off,
746 				PATTERN_SRC | PATTERN_COPY, false, is_memset);
747 
748 		error_count += dmatest_verify(thread->dsts, dst_off + len,
749 				params->buf_size, dst_off + len,
750 				PATTERN_DST, false, is_memset);
751 
752 		diff = ktime_sub(ktime_get(), start);
753 		comparetime = ktime_add(comparetime, diff);
754 
755 		if (error_count) {
756 			result("data error", total_tests, src_off, dst_off,
757 			       len, error_count);
758 			failed_tests++;
759 		} else {
760 			verbose_result("test passed", total_tests, src_off,
761 				       dst_off, len, 0);
762 		}
763 	}
764 	ktime = ktime_sub(ktime_get(), ktime);
765 	ktime = ktime_sub(ktime, comparetime);
766 	ktime = ktime_sub(ktime, filltime);
767 	runtime = ktime_to_us(ktime);
768 
769 	ret = 0;
770 err_dstbuf:
771 	for (i = 0; thread->udsts[i]; i++)
772 		kfree(thread->udsts[i]);
773 	kfree(thread->udsts);
774 err_udsts:
775 	kfree(thread->dsts);
776 err_dsts:
777 err_srcbuf:
778 	for (i = 0; thread->usrcs[i]; i++)
779 		kfree(thread->usrcs[i]);
780 	kfree(thread->usrcs);
781 err_usrcs:
782 	kfree(thread->srcs);
783 err_srcs:
784 	kfree(pq_coefs);
785 err_thread_type:
786 	pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
787 		current->comm, total_tests, failed_tests,
788 		dmatest_persec(runtime, total_tests),
789 		dmatest_KBs(runtime, total_len), ret);
790 
791 	/* terminate all transfers on specified channels */
792 	if (ret)
793 		dmaengine_terminate_all(chan);
794 
795 	thread->done = true;
796 	wake_up(&thread_wait);
797 
798 	return ret;
799 }
800 
801 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
802 {
803 	struct dmatest_thread	*thread;
804 	struct dmatest_thread	*_thread;
805 	int			ret;
806 
807 	list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
808 		ret = kthread_stop(thread->task);
809 		pr_debug("thread %s exited with status %d\n",
810 			 thread->task->comm, ret);
811 		list_del(&thread->node);
812 		put_task_struct(thread->task);
813 		kfree(thread);
814 	}
815 
816 	/* terminate all transfers on specified channels */
817 	dmaengine_terminate_all(dtc->chan);
818 
819 	kfree(dtc);
820 }
821 
822 static int dmatest_add_threads(struct dmatest_info *info,
823 		struct dmatest_chan *dtc, enum dma_transaction_type type)
824 {
825 	struct dmatest_params *params = &info->params;
826 	struct dmatest_thread *thread;
827 	struct dma_chan *chan = dtc->chan;
828 	char *op;
829 	unsigned int i;
830 
831 	if (type == DMA_MEMCPY)
832 		op = "copy";
833 	else if (type == DMA_MEMSET)
834 		op = "set";
835 	else if (type == DMA_XOR)
836 		op = "xor";
837 	else if (type == DMA_PQ)
838 		op = "pq";
839 	else
840 		return -EINVAL;
841 
842 	for (i = 0; i < params->threads_per_chan; i++) {
843 		thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
844 		if (!thread) {
845 			pr_warn("No memory for %s-%s%u\n",
846 				dma_chan_name(chan), op, i);
847 			break;
848 		}
849 		thread->info = info;
850 		thread->chan = dtc->chan;
851 		thread->type = type;
852 		smp_wmb();
853 		thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
854 				dma_chan_name(chan), op, i);
855 		if (IS_ERR(thread->task)) {
856 			pr_warn("Failed to create thread %s-%s%u\n",
857 				dma_chan_name(chan), op, i);
858 			kfree(thread);
859 			break;
860 		}
861 
862 		/* srcbuf and dstbuf are allocated by the thread itself */
863 		get_task_struct(thread->task);
864 		list_add_tail(&thread->node, &dtc->threads);
865 		wake_up_process(thread->task);
866 	}
867 
868 	return i;
869 }
870 
871 static int dmatest_add_channel(struct dmatest_info *info,
872 		struct dma_chan *chan)
873 {
874 	struct dmatest_chan	*dtc;
875 	struct dma_device	*dma_dev = chan->device;
876 	unsigned int		thread_count = 0;
877 	int cnt;
878 
879 	dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
880 	if (!dtc) {
881 		pr_warn("No memory for %s\n", dma_chan_name(chan));
882 		return -ENOMEM;
883 	}
884 
885 	dtc->chan = chan;
886 	INIT_LIST_HEAD(&dtc->threads);
887 
888 	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
889 		if (dmatest == 0) {
890 			cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
891 			thread_count += cnt > 0 ? cnt : 0;
892 		}
893 	}
894 
895 	if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
896 		if (dmatest == 1) {
897 			cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
898 			thread_count += cnt > 0 ? cnt : 0;
899 		}
900 	}
901 
902 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
903 		cnt = dmatest_add_threads(info, dtc, DMA_XOR);
904 		thread_count += cnt > 0 ? cnt : 0;
905 	}
906 	if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
907 		cnt = dmatest_add_threads(info, dtc, DMA_PQ);
908 		thread_count += cnt > 0 ? cnt : 0;
909 	}
910 
911 	pr_info("Started %u threads using %s\n",
912 		thread_count, dma_chan_name(chan));
913 
914 	list_add_tail(&dtc->node, &info->channels);
915 	info->nr_channels++;
916 
917 	return 0;
918 }
919 
920 static bool filter(struct dma_chan *chan, void *param)
921 {
922 	struct dmatest_params *params = param;
923 
924 	if (!dmatest_match_channel(params, chan) ||
925 	    !dmatest_match_device(params, chan->device))
926 		return false;
927 	else
928 		return true;
929 }
930 
931 static void request_channels(struct dmatest_info *info,
932 			     enum dma_transaction_type type)
933 {
934 	dma_cap_mask_t mask;
935 
936 	dma_cap_zero(mask);
937 	dma_cap_set(type, mask);
938 	for (;;) {
939 		struct dmatest_params *params = &info->params;
940 		struct dma_chan *chan;
941 
942 		chan = dma_request_channel(mask, filter, params);
943 		if (chan) {
944 			if (dmatest_add_channel(info, chan)) {
945 				dma_release_channel(chan);
946 				break; /* add_channel failed, punt */
947 			}
948 		} else
949 			break; /* no more channels available */
950 		if (params->max_channels &&
951 		    info->nr_channels >= params->max_channels)
952 			break; /* we have all we need */
953 	}
954 }
955 
956 static void run_threaded_test(struct dmatest_info *info)
957 {
958 	struct dmatest_params *params = &info->params;
959 
960 	/* Copy test parameters */
961 	params->buf_size = test_buf_size;
962 	strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
963 	strlcpy(params->device, strim(test_device), sizeof(params->device));
964 	params->threads_per_chan = threads_per_chan;
965 	params->max_channels = max_channels;
966 	params->iterations = iterations;
967 	params->xor_sources = xor_sources;
968 	params->pq_sources = pq_sources;
969 	params->timeout = timeout;
970 	params->noverify = noverify;
971 
972 	request_channels(info, DMA_MEMCPY);
973 	request_channels(info, DMA_MEMSET);
974 	request_channels(info, DMA_XOR);
975 	request_channels(info, DMA_PQ);
976 }
977 
978 static void stop_threaded_test(struct dmatest_info *info)
979 {
980 	struct dmatest_chan *dtc, *_dtc;
981 	struct dma_chan *chan;
982 
983 	list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
984 		list_del(&dtc->node);
985 		chan = dtc->chan;
986 		dmatest_cleanup_channel(dtc);
987 		pr_debug("dropped channel %s\n", dma_chan_name(chan));
988 		dma_release_channel(chan);
989 	}
990 
991 	info->nr_channels = 0;
992 }
993 
994 static void restart_threaded_test(struct dmatest_info *info, bool run)
995 {
996 	/* we might be called early to set run=, defer running until all
997 	 * parameters have been evaluated
998 	 */
999 	if (!info->did_init)
1000 		return;
1001 
1002 	/* Stop any running test first */
1003 	stop_threaded_test(info);
1004 
1005 	/* Run test with new parameters */
1006 	run_threaded_test(info);
1007 }
1008 
1009 static int dmatest_run_get(char *val, const struct kernel_param *kp)
1010 {
1011 	struct dmatest_info *info = &test_info;
1012 
1013 	mutex_lock(&info->lock);
1014 	if (is_threaded_test_run(info)) {
1015 		dmatest_run = true;
1016 	} else {
1017 		stop_threaded_test(info);
1018 		dmatest_run = false;
1019 	}
1020 	mutex_unlock(&info->lock);
1021 
1022 	return param_get_bool(val, kp);
1023 }
1024 
1025 static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1026 {
1027 	struct dmatest_info *info = &test_info;
1028 	int ret;
1029 
1030 	mutex_lock(&info->lock);
1031 	ret = param_set_bool(val, kp);
1032 	if (ret) {
1033 		mutex_unlock(&info->lock);
1034 		return ret;
1035 	}
1036 
1037 	if (is_threaded_test_run(info))
1038 		ret = -EBUSY;
1039 	else if (dmatest_run)
1040 		restart_threaded_test(info, dmatest_run);
1041 
1042 	mutex_unlock(&info->lock);
1043 
1044 	return ret;
1045 }
1046 
1047 static int __init dmatest_init(void)
1048 {
1049 	struct dmatest_info *info = &test_info;
1050 	struct dmatest_params *params = &info->params;
1051 
1052 	if (dmatest_run) {
1053 		mutex_lock(&info->lock);
1054 		run_threaded_test(info);
1055 		mutex_unlock(&info->lock);
1056 	}
1057 
1058 	if (params->iterations && wait)
1059 		wait_event(thread_wait, !is_threaded_test_run(info));
1060 
1061 	/* module parameters are stable, inittime tests are started,
1062 	 * let userspace take over 'run' control
1063 	 */
1064 	info->did_init = true;
1065 
1066 	return 0;
1067 }
1068 /* when compiled-in wait for drivers to load first */
1069 late_initcall(dmatest_init);
1070 
1071 static void __exit dmatest_exit(void)
1072 {
1073 	struct dmatest_info *info = &test_info;
1074 
1075 	mutex_lock(&info->lock);
1076 	stop_threaded_test(info);
1077 	mutex_unlock(&info->lock);
1078 }
1079 module_exit(dmatest_exit);
1080 
1081 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1082 MODULE_LICENSE("GPL v2");
1083