xref: /openbmc/linux/drivers/dma/dmatest.c (revision ddc141e5)
1 /*
2  * DMA Engine test module
3  *
4  * Copyright (C) 2007 Atmel Corporation
5  * Copyright (C) 2013 Intel Corporation
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 
13 #include <linux/delay.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dmaengine.h>
16 #include <linux/freezer.h>
17 #include <linux/init.h>
18 #include <linux/kthread.h>
19 #include <linux/sched/task.h>
20 #include <linux/module.h>
21 #include <linux/moduleparam.h>
22 #include <linux/random.h>
23 #include <linux/slab.h>
24 #include <linux/wait.h>
25 
26 static unsigned int test_buf_size = 16384;
27 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
28 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
29 
30 static char test_channel[20];
31 module_param_string(channel, test_channel, sizeof(test_channel),
32 		S_IRUGO | S_IWUSR);
33 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
34 
35 static char test_device[32];
36 module_param_string(device, test_device, sizeof(test_device),
37 		S_IRUGO | S_IWUSR);
38 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
39 
40 static unsigned int threads_per_chan = 1;
41 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
42 MODULE_PARM_DESC(threads_per_chan,
43 		"Number of threads to start per channel (default: 1)");
44 
45 static unsigned int max_channels;
46 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
47 MODULE_PARM_DESC(max_channels,
48 		"Maximum number of channels to use (default: all)");
49 
50 static unsigned int iterations;
51 module_param(iterations, uint, S_IRUGO | S_IWUSR);
52 MODULE_PARM_DESC(iterations,
53 		"Iterations before stopping test (default: infinite)");
54 
55 static unsigned int dmatest;
56 module_param(dmatest, uint, S_IRUGO | S_IWUSR);
57 MODULE_PARM_DESC(dmatest,
58 		"dmatest 0-memcpy 1-memset (default: 0)");
59 
60 static unsigned int xor_sources = 3;
61 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
62 MODULE_PARM_DESC(xor_sources,
63 		"Number of xor source buffers (default: 3)");
64 
65 static unsigned int pq_sources = 3;
66 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
67 MODULE_PARM_DESC(pq_sources,
68 		"Number of p+q source buffers (default: 3)");
69 
70 static int timeout = 3000;
71 module_param(timeout, uint, S_IRUGO | S_IWUSR);
72 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
73 		 "Pass -1 for infinite timeout");
74 
75 static bool noverify;
76 module_param(noverify, bool, S_IRUGO | S_IWUSR);
77 MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
78 
79 static bool verbose;
80 module_param(verbose, bool, S_IRUGO | S_IWUSR);
81 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
82 
83 /**
84  * struct dmatest_params - test parameters.
85  * @buf_size:		size of the memcpy test buffer
86  * @channel:		bus ID of the channel to test
87  * @device:		bus ID of the DMA Engine to test
88  * @threads_per_chan:	number of threads to start per channel
89  * @max_channels:	maximum number of channels to use
90  * @iterations:		iterations before stopping test
91  * @xor_sources:	number of xor source buffers
92  * @pq_sources:		number of p+q source buffers
93  * @timeout:		transfer timeout in msec, -1 for infinite timeout
94  */
95 struct dmatest_params {
96 	unsigned int	buf_size;
97 	char		channel[20];
98 	char		device[32];
99 	unsigned int	threads_per_chan;
100 	unsigned int	max_channels;
101 	unsigned int	iterations;
102 	unsigned int	xor_sources;
103 	unsigned int	pq_sources;
104 	int		timeout;
105 	bool		noverify;
106 };
107 
108 /**
109  * struct dmatest_info - test information.
110  * @params:		test parameters
111  * @lock:		access protection to the fields of this structure
112  */
113 static struct dmatest_info {
114 	/* Test parameters */
115 	struct dmatest_params	params;
116 
117 	/* Internal state */
118 	struct list_head	channels;
119 	unsigned int		nr_channels;
120 	struct mutex		lock;
121 	bool			did_init;
122 } test_info = {
123 	.channels = LIST_HEAD_INIT(test_info.channels),
124 	.lock = __MUTEX_INITIALIZER(test_info.lock),
125 };
126 
127 static int dmatest_run_set(const char *val, const struct kernel_param *kp);
128 static int dmatest_run_get(char *val, const struct kernel_param *kp);
129 static const struct kernel_param_ops run_ops = {
130 	.set = dmatest_run_set,
131 	.get = dmatest_run_get,
132 };
133 static bool dmatest_run;
134 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
135 MODULE_PARM_DESC(run, "Run the test (default: false)");
136 
137 /* Maximum amount of mismatched bytes in buffer to print */
138 #define MAX_ERROR_COUNT		32
139 
140 /*
141  * Initialization patterns. All bytes in the source buffer has bit 7
142  * set, all bytes in the destination buffer has bit 7 cleared.
143  *
144  * Bit 6 is set for all bytes which are to be copied by the DMA
145  * engine. Bit 5 is set for all bytes which are to be overwritten by
146  * the DMA engine.
147  *
148  * The remaining bits are the inverse of a counter which increments by
149  * one for each byte address.
150  */
151 #define PATTERN_SRC		0x80
152 #define PATTERN_DST		0x00
153 #define PATTERN_COPY		0x40
154 #define PATTERN_OVERWRITE	0x20
155 #define PATTERN_COUNT_MASK	0x1f
156 #define PATTERN_MEMSET_IDX	0x01
157 
158 /* poor man's completion - we want to use wait_event_freezable() on it */
159 struct dmatest_done {
160 	bool			done;
161 	wait_queue_head_t	*wait;
162 };
163 
164 struct dmatest_thread {
165 	struct list_head	node;
166 	struct dmatest_info	*info;
167 	struct task_struct	*task;
168 	struct dma_chan		*chan;
169 	u8			**srcs;
170 	u8			**usrcs;
171 	u8			**dsts;
172 	u8			**udsts;
173 	enum dma_transaction_type type;
174 	wait_queue_head_t done_wait;
175 	struct dmatest_done test_done;
176 	bool			done;
177 };
178 
179 struct dmatest_chan {
180 	struct list_head	node;
181 	struct dma_chan		*chan;
182 	struct list_head	threads;
183 };
184 
185 static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
186 static bool wait;
187 
188 static bool is_threaded_test_run(struct dmatest_info *info)
189 {
190 	struct dmatest_chan *dtc;
191 
192 	list_for_each_entry(dtc, &info->channels, node) {
193 		struct dmatest_thread *thread;
194 
195 		list_for_each_entry(thread, &dtc->threads, node) {
196 			if (!thread->done)
197 				return true;
198 		}
199 	}
200 
201 	return false;
202 }
203 
204 static int dmatest_wait_get(char *val, const struct kernel_param *kp)
205 {
206 	struct dmatest_info *info = &test_info;
207 	struct dmatest_params *params = &info->params;
208 
209 	if (params->iterations)
210 		wait_event(thread_wait, !is_threaded_test_run(info));
211 	wait = true;
212 	return param_get_bool(val, kp);
213 }
214 
215 static const struct kernel_param_ops wait_ops = {
216 	.get = dmatest_wait_get,
217 	.set = param_set_bool,
218 };
219 module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
220 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
221 
222 static bool dmatest_match_channel(struct dmatest_params *params,
223 		struct dma_chan *chan)
224 {
225 	if (params->channel[0] == '\0')
226 		return true;
227 	return strcmp(dma_chan_name(chan), params->channel) == 0;
228 }
229 
230 static bool dmatest_match_device(struct dmatest_params *params,
231 		struct dma_device *device)
232 {
233 	if (params->device[0] == '\0')
234 		return true;
235 	return strcmp(dev_name(device->dev), params->device) == 0;
236 }
237 
238 static unsigned long dmatest_random(void)
239 {
240 	unsigned long buf;
241 
242 	prandom_bytes(&buf, sizeof(buf));
243 	return buf;
244 }
245 
246 static inline u8 gen_inv_idx(u8 index, bool is_memset)
247 {
248 	u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
249 
250 	return ~val & PATTERN_COUNT_MASK;
251 }
252 
253 static inline u8 gen_src_value(u8 index, bool is_memset)
254 {
255 	return PATTERN_SRC | gen_inv_idx(index, is_memset);
256 }
257 
258 static inline u8 gen_dst_value(u8 index, bool is_memset)
259 {
260 	return PATTERN_DST | gen_inv_idx(index, is_memset);
261 }
262 
263 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
264 		unsigned int buf_size, bool is_memset)
265 {
266 	unsigned int i;
267 	u8 *buf;
268 
269 	for (; (buf = *bufs); bufs++) {
270 		for (i = 0; i < start; i++)
271 			buf[i] = gen_src_value(i, is_memset);
272 		for ( ; i < start + len; i++)
273 			buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
274 		for ( ; i < buf_size; i++)
275 			buf[i] = gen_src_value(i, is_memset);
276 		buf++;
277 	}
278 }
279 
280 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
281 		unsigned int buf_size, bool is_memset)
282 {
283 	unsigned int i;
284 	u8 *buf;
285 
286 	for (; (buf = *bufs); bufs++) {
287 		for (i = 0; i < start; i++)
288 			buf[i] = gen_dst_value(i, is_memset);
289 		for ( ; i < start + len; i++)
290 			buf[i] = gen_dst_value(i, is_memset) |
291 						PATTERN_OVERWRITE;
292 		for ( ; i < buf_size; i++)
293 			buf[i] = gen_dst_value(i, is_memset);
294 	}
295 }
296 
297 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
298 		unsigned int counter, bool is_srcbuf, bool is_memset)
299 {
300 	u8		diff = actual ^ pattern;
301 	u8		expected = pattern | gen_inv_idx(counter, is_memset);
302 	const char	*thread_name = current->comm;
303 
304 	if (is_srcbuf)
305 		pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
306 			thread_name, index, expected, actual);
307 	else if ((pattern & PATTERN_COPY)
308 			&& (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
309 		pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
310 			thread_name, index, expected, actual);
311 	else if (diff & PATTERN_SRC)
312 		pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
313 			thread_name, index, expected, actual);
314 	else
315 		pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
316 			thread_name, index, expected, actual);
317 }
318 
319 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
320 		unsigned int end, unsigned int counter, u8 pattern,
321 		bool is_srcbuf, bool is_memset)
322 {
323 	unsigned int i;
324 	unsigned int error_count = 0;
325 	u8 actual;
326 	u8 expected;
327 	u8 *buf;
328 	unsigned int counter_orig = counter;
329 
330 	for (; (buf = *bufs); bufs++) {
331 		counter = counter_orig;
332 		for (i = start; i < end; i++) {
333 			actual = buf[i];
334 			expected = pattern | gen_inv_idx(counter, is_memset);
335 			if (actual != expected) {
336 				if (error_count < MAX_ERROR_COUNT)
337 					dmatest_mismatch(actual, pattern, i,
338 							 counter, is_srcbuf,
339 							 is_memset);
340 				error_count++;
341 			}
342 			counter++;
343 		}
344 	}
345 
346 	if (error_count > MAX_ERROR_COUNT)
347 		pr_warn("%s: %u errors suppressed\n",
348 			current->comm, error_count - MAX_ERROR_COUNT);
349 
350 	return error_count;
351 }
352 
353 
354 static void dmatest_callback(void *arg)
355 {
356 	struct dmatest_done *done = arg;
357 	struct dmatest_thread *thread =
358 		container_of(done, struct dmatest_thread, test_done);
359 	if (!thread->done) {
360 		done->done = true;
361 		wake_up_all(done->wait);
362 	} else {
363 		/*
364 		 * If thread->done, it means that this callback occurred
365 		 * after the parent thread has cleaned up. This can
366 		 * happen in the case that driver doesn't implement
367 		 * the terminate_all() functionality and a dma operation
368 		 * did not occur within the timeout period
369 		 */
370 		WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
371 	}
372 }
373 
374 static unsigned int min_odd(unsigned int x, unsigned int y)
375 {
376 	unsigned int val = min(x, y);
377 
378 	return val % 2 ? val : val - 1;
379 }
380 
381 static void result(const char *err, unsigned int n, unsigned int src_off,
382 		   unsigned int dst_off, unsigned int len, unsigned long data)
383 {
384 	pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
385 		current->comm, n, err, src_off, dst_off, len, data);
386 }
387 
388 static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
389 		       unsigned int dst_off, unsigned int len,
390 		       unsigned long data)
391 {
392 	pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
393 		 current->comm, n, err, src_off, dst_off, len, data);
394 }
395 
396 #define verbose_result(err, n, src_off, dst_off, len, data) ({	\
397 	if (verbose)						\
398 		result(err, n, src_off, dst_off, len, data);	\
399 	else							\
400 		dbg_result(err, n, src_off, dst_off, len, data);\
401 })
402 
403 static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
404 {
405 	unsigned long long per_sec = 1000000;
406 
407 	if (runtime <= 0)
408 		return 0;
409 
410 	/* drop precision until runtime is 32-bits */
411 	while (runtime > UINT_MAX) {
412 		runtime >>= 1;
413 		per_sec <<= 1;
414 	}
415 
416 	per_sec *= val;
417 	do_div(per_sec, runtime);
418 	return per_sec;
419 }
420 
421 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
422 {
423 	return dmatest_persec(runtime, len >> 10);
424 }
425 
426 /*
427  * This function repeatedly tests DMA transfers of various lengths and
428  * offsets for a given operation type until it is told to exit by
429  * kthread_stop(). There may be multiple threads running this function
430  * in parallel for a single channel, and there may be multiple channels
431  * being tested in parallel.
432  *
433  * Before each test, the source and destination buffer is initialized
434  * with a known pattern. This pattern is different depending on
435  * whether it's in an area which is supposed to be copied or
436  * overwritten, and different in the source and destination buffers.
437  * So if the DMA engine doesn't copy exactly what we tell it to copy,
438  * we'll notice.
439  */
440 static int dmatest_func(void *data)
441 {
442 	struct dmatest_thread	*thread = data;
443 	struct dmatest_done	*done = &thread->test_done;
444 	struct dmatest_info	*info;
445 	struct dmatest_params	*params;
446 	struct dma_chan		*chan;
447 	struct dma_device	*dev;
448 	unsigned int		error_count;
449 	unsigned int		failed_tests = 0;
450 	unsigned int		total_tests = 0;
451 	dma_cookie_t		cookie;
452 	enum dma_status		status;
453 	enum dma_ctrl_flags 	flags;
454 	u8			*pq_coefs = NULL;
455 	int			ret;
456 	int			src_cnt;
457 	int			dst_cnt;
458 	int			i;
459 	ktime_t			ktime, start, diff;
460 	ktime_t			filltime = 0;
461 	ktime_t			comparetime = 0;
462 	s64			runtime = 0;
463 	unsigned long long	total_len = 0;
464 	u8			align = 0;
465 	bool			is_memset = false;
466 
467 	set_freezable();
468 
469 	ret = -ENOMEM;
470 
471 	smp_rmb();
472 	info = thread->info;
473 	params = &info->params;
474 	chan = thread->chan;
475 	dev = chan->device;
476 	if (thread->type == DMA_MEMCPY) {
477 		align = dev->copy_align;
478 		src_cnt = dst_cnt = 1;
479 	} else if (thread->type == DMA_MEMSET) {
480 		align = dev->fill_align;
481 		src_cnt = dst_cnt = 1;
482 		is_memset = true;
483 	} else if (thread->type == DMA_XOR) {
484 		/* force odd to ensure dst = src */
485 		src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
486 		dst_cnt = 1;
487 		align = dev->xor_align;
488 	} else if (thread->type == DMA_PQ) {
489 		/* force odd to ensure dst = src */
490 		src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
491 		dst_cnt = 2;
492 		align = dev->pq_align;
493 
494 		pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
495 		if (!pq_coefs)
496 			goto err_thread_type;
497 
498 		for (i = 0; i < src_cnt; i++)
499 			pq_coefs[i] = 1;
500 	} else
501 		goto err_thread_type;
502 
503 	thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
504 	if (!thread->srcs)
505 		goto err_srcs;
506 
507 	thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
508 	if (!thread->usrcs)
509 		goto err_usrcs;
510 
511 	for (i = 0; i < src_cnt; i++) {
512 		thread->usrcs[i] = kmalloc(params->buf_size + align,
513 					   GFP_KERNEL);
514 		if (!thread->usrcs[i])
515 			goto err_srcbuf;
516 
517 		/* align srcs to alignment restriction */
518 		if (align)
519 			thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align);
520 		else
521 			thread->srcs[i] = thread->usrcs[i];
522 	}
523 	thread->srcs[i] = NULL;
524 
525 	thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
526 	if (!thread->dsts)
527 		goto err_dsts;
528 
529 	thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
530 	if (!thread->udsts)
531 		goto err_udsts;
532 
533 	for (i = 0; i < dst_cnt; i++) {
534 		thread->udsts[i] = kmalloc(params->buf_size + align,
535 					   GFP_KERNEL);
536 		if (!thread->udsts[i])
537 			goto err_dstbuf;
538 
539 		/* align dsts to alignment restriction */
540 		if (align)
541 			thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align);
542 		else
543 			thread->dsts[i] = thread->udsts[i];
544 	}
545 	thread->dsts[i] = NULL;
546 
547 	set_user_nice(current, 10);
548 
549 	/*
550 	 * src and dst buffers are freed by ourselves below
551 	 */
552 	flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
553 
554 	ktime = ktime_get();
555 	while (!kthread_should_stop()
556 	       && !(params->iterations && total_tests >= params->iterations)) {
557 		struct dma_async_tx_descriptor *tx = NULL;
558 		struct dmaengine_unmap_data *um;
559 		dma_addr_t srcs[src_cnt];
560 		dma_addr_t *dsts;
561 		unsigned int src_off, dst_off, len;
562 
563 		total_tests++;
564 
565 		/* Check if buffer count fits into map count variable (u8) */
566 		if ((src_cnt + dst_cnt) >= 255) {
567 			pr_err("too many buffers (%d of 255 supported)\n",
568 			       src_cnt + dst_cnt);
569 			break;
570 		}
571 
572 		if (1 << align > params->buf_size) {
573 			pr_err("%u-byte buffer too small for %d-byte alignment\n",
574 			       params->buf_size, 1 << align);
575 			break;
576 		}
577 
578 		if (params->noverify)
579 			len = params->buf_size;
580 		else
581 			len = dmatest_random() % params->buf_size + 1;
582 
583 		len = (len >> align) << align;
584 		if (!len)
585 			len = 1 << align;
586 
587 		total_len += len;
588 
589 		if (params->noverify) {
590 			src_off = 0;
591 			dst_off = 0;
592 		} else {
593 			start = ktime_get();
594 			src_off = dmatest_random() % (params->buf_size - len + 1);
595 			dst_off = dmatest_random() % (params->buf_size - len + 1);
596 
597 			src_off = (src_off >> align) << align;
598 			dst_off = (dst_off >> align) << align;
599 
600 			dmatest_init_srcs(thread->srcs, src_off, len,
601 					  params->buf_size, is_memset);
602 			dmatest_init_dsts(thread->dsts, dst_off, len,
603 					  params->buf_size, is_memset);
604 
605 			diff = ktime_sub(ktime_get(), start);
606 			filltime = ktime_add(filltime, diff);
607 		}
608 
609 		um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt,
610 					      GFP_KERNEL);
611 		if (!um) {
612 			failed_tests++;
613 			result("unmap data NULL", total_tests,
614 			       src_off, dst_off, len, ret);
615 			continue;
616 		}
617 
618 		um->len = params->buf_size;
619 		for (i = 0; i < src_cnt; i++) {
620 			void *buf = thread->srcs[i];
621 			struct page *pg = virt_to_page(buf);
622 			unsigned long pg_off = offset_in_page(buf);
623 
624 			um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
625 						   um->len, DMA_TO_DEVICE);
626 			srcs[i] = um->addr[i] + src_off;
627 			ret = dma_mapping_error(dev->dev, um->addr[i]);
628 			if (ret) {
629 				dmaengine_unmap_put(um);
630 				result("src mapping error", total_tests,
631 				       src_off, dst_off, len, ret);
632 				failed_tests++;
633 				continue;
634 			}
635 			um->to_cnt++;
636 		}
637 		/* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
638 		dsts = &um->addr[src_cnt];
639 		for (i = 0; i < dst_cnt; i++) {
640 			void *buf = thread->dsts[i];
641 			struct page *pg = virt_to_page(buf);
642 			unsigned long pg_off = offset_in_page(buf);
643 
644 			dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
645 					       DMA_BIDIRECTIONAL);
646 			ret = dma_mapping_error(dev->dev, dsts[i]);
647 			if (ret) {
648 				dmaengine_unmap_put(um);
649 				result("dst mapping error", total_tests,
650 				       src_off, dst_off, len, ret);
651 				failed_tests++;
652 				continue;
653 			}
654 			um->bidi_cnt++;
655 		}
656 
657 		if (thread->type == DMA_MEMCPY)
658 			tx = dev->device_prep_dma_memcpy(chan,
659 							 dsts[0] + dst_off,
660 							 srcs[0], len, flags);
661 		else if (thread->type == DMA_MEMSET)
662 			tx = dev->device_prep_dma_memset(chan,
663 						dsts[0] + dst_off,
664 						*(thread->srcs[0] + src_off),
665 						len, flags);
666 		else if (thread->type == DMA_XOR)
667 			tx = dev->device_prep_dma_xor(chan,
668 						      dsts[0] + dst_off,
669 						      srcs, src_cnt,
670 						      len, flags);
671 		else if (thread->type == DMA_PQ) {
672 			dma_addr_t dma_pq[dst_cnt];
673 
674 			for (i = 0; i < dst_cnt; i++)
675 				dma_pq[i] = dsts[i] + dst_off;
676 			tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
677 						     src_cnt, pq_coefs,
678 						     len, flags);
679 		}
680 
681 		if (!tx) {
682 			dmaengine_unmap_put(um);
683 			result("prep error", total_tests, src_off,
684 			       dst_off, len, ret);
685 			msleep(100);
686 			failed_tests++;
687 			continue;
688 		}
689 
690 		done->done = false;
691 		tx->callback = dmatest_callback;
692 		tx->callback_param = done;
693 		cookie = tx->tx_submit(tx);
694 
695 		if (dma_submit_error(cookie)) {
696 			dmaengine_unmap_put(um);
697 			result("submit error", total_tests, src_off,
698 			       dst_off, len, ret);
699 			msleep(100);
700 			failed_tests++;
701 			continue;
702 		}
703 		dma_async_issue_pending(chan);
704 
705 		wait_event_freezable_timeout(thread->done_wait, done->done,
706 					     msecs_to_jiffies(params->timeout));
707 
708 		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
709 
710 		if (!done->done) {
711 			dmaengine_unmap_put(um);
712 			result("test timed out", total_tests, src_off, dst_off,
713 			       len, 0);
714 			failed_tests++;
715 			continue;
716 		} else if (status != DMA_COMPLETE) {
717 			dmaengine_unmap_put(um);
718 			result(status == DMA_ERROR ?
719 			       "completion error status" :
720 			       "completion busy status", total_tests, src_off,
721 			       dst_off, len, ret);
722 			failed_tests++;
723 			continue;
724 		}
725 
726 		dmaengine_unmap_put(um);
727 
728 		if (params->noverify) {
729 			verbose_result("test passed", total_tests, src_off,
730 				       dst_off, len, 0);
731 			continue;
732 		}
733 
734 		start = ktime_get();
735 		pr_debug("%s: verifying source buffer...\n", current->comm);
736 		error_count = dmatest_verify(thread->srcs, 0, src_off,
737 				0, PATTERN_SRC, true, is_memset);
738 		error_count += dmatest_verify(thread->srcs, src_off,
739 				src_off + len, src_off,
740 				PATTERN_SRC | PATTERN_COPY, true, is_memset);
741 		error_count += dmatest_verify(thread->srcs, src_off + len,
742 				params->buf_size, src_off + len,
743 				PATTERN_SRC, true, is_memset);
744 
745 		pr_debug("%s: verifying dest buffer...\n", current->comm);
746 		error_count += dmatest_verify(thread->dsts, 0, dst_off,
747 				0, PATTERN_DST, false, is_memset);
748 
749 		error_count += dmatest_verify(thread->dsts, dst_off,
750 				dst_off + len, src_off,
751 				PATTERN_SRC | PATTERN_COPY, false, is_memset);
752 
753 		error_count += dmatest_verify(thread->dsts, dst_off + len,
754 				params->buf_size, dst_off + len,
755 				PATTERN_DST, false, is_memset);
756 
757 		diff = ktime_sub(ktime_get(), start);
758 		comparetime = ktime_add(comparetime, diff);
759 
760 		if (error_count) {
761 			result("data error", total_tests, src_off, dst_off,
762 			       len, error_count);
763 			failed_tests++;
764 		} else {
765 			verbose_result("test passed", total_tests, src_off,
766 				       dst_off, len, 0);
767 		}
768 	}
769 	ktime = ktime_sub(ktime_get(), ktime);
770 	ktime = ktime_sub(ktime, comparetime);
771 	ktime = ktime_sub(ktime, filltime);
772 	runtime = ktime_to_us(ktime);
773 
774 	ret = 0;
775 err_dstbuf:
776 	for (i = 0; thread->udsts[i]; i++)
777 		kfree(thread->udsts[i]);
778 	kfree(thread->udsts);
779 err_udsts:
780 	kfree(thread->dsts);
781 err_dsts:
782 err_srcbuf:
783 	for (i = 0; thread->usrcs[i]; i++)
784 		kfree(thread->usrcs[i]);
785 	kfree(thread->usrcs);
786 err_usrcs:
787 	kfree(thread->srcs);
788 err_srcs:
789 	kfree(pq_coefs);
790 err_thread_type:
791 	pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
792 		current->comm, total_tests, failed_tests,
793 		dmatest_persec(runtime, total_tests),
794 		dmatest_KBs(runtime, total_len), ret);
795 
796 	/* terminate all transfers on specified channels */
797 	if (ret || failed_tests)
798 		dmaengine_terminate_all(chan);
799 
800 	thread->done = true;
801 	wake_up(&thread_wait);
802 
803 	return ret;
804 }
805 
806 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
807 {
808 	struct dmatest_thread	*thread;
809 	struct dmatest_thread	*_thread;
810 	int			ret;
811 
812 	list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
813 		ret = kthread_stop(thread->task);
814 		pr_debug("thread %s exited with status %d\n",
815 			 thread->task->comm, ret);
816 		list_del(&thread->node);
817 		put_task_struct(thread->task);
818 		kfree(thread);
819 	}
820 
821 	/* terminate all transfers on specified channels */
822 	dmaengine_terminate_all(dtc->chan);
823 
824 	kfree(dtc);
825 }
826 
827 static int dmatest_add_threads(struct dmatest_info *info,
828 		struct dmatest_chan *dtc, enum dma_transaction_type type)
829 {
830 	struct dmatest_params *params = &info->params;
831 	struct dmatest_thread *thread;
832 	struct dma_chan *chan = dtc->chan;
833 	char *op;
834 	unsigned int i;
835 
836 	if (type == DMA_MEMCPY)
837 		op = "copy";
838 	else if (type == DMA_MEMSET)
839 		op = "set";
840 	else if (type == DMA_XOR)
841 		op = "xor";
842 	else if (type == DMA_PQ)
843 		op = "pq";
844 	else
845 		return -EINVAL;
846 
847 	for (i = 0; i < params->threads_per_chan; i++) {
848 		thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
849 		if (!thread) {
850 			pr_warn("No memory for %s-%s%u\n",
851 				dma_chan_name(chan), op, i);
852 			break;
853 		}
854 		thread->info = info;
855 		thread->chan = dtc->chan;
856 		thread->type = type;
857 		thread->test_done.wait = &thread->done_wait;
858 		init_waitqueue_head(&thread->done_wait);
859 		smp_wmb();
860 		thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
861 				dma_chan_name(chan), op, i);
862 		if (IS_ERR(thread->task)) {
863 			pr_warn("Failed to create thread %s-%s%u\n",
864 				dma_chan_name(chan), op, i);
865 			kfree(thread);
866 			break;
867 		}
868 
869 		/* srcbuf and dstbuf are allocated by the thread itself */
870 		get_task_struct(thread->task);
871 		list_add_tail(&thread->node, &dtc->threads);
872 		wake_up_process(thread->task);
873 	}
874 
875 	return i;
876 }
877 
878 static int dmatest_add_channel(struct dmatest_info *info,
879 		struct dma_chan *chan)
880 {
881 	struct dmatest_chan	*dtc;
882 	struct dma_device	*dma_dev = chan->device;
883 	unsigned int		thread_count = 0;
884 	int cnt;
885 
886 	dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
887 	if (!dtc) {
888 		pr_warn("No memory for %s\n", dma_chan_name(chan));
889 		return -ENOMEM;
890 	}
891 
892 	dtc->chan = chan;
893 	INIT_LIST_HEAD(&dtc->threads);
894 
895 	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
896 		if (dmatest == 0) {
897 			cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
898 			thread_count += cnt > 0 ? cnt : 0;
899 		}
900 	}
901 
902 	if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
903 		if (dmatest == 1) {
904 			cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
905 			thread_count += cnt > 0 ? cnt : 0;
906 		}
907 	}
908 
909 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
910 		cnt = dmatest_add_threads(info, dtc, DMA_XOR);
911 		thread_count += cnt > 0 ? cnt : 0;
912 	}
913 	if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
914 		cnt = dmatest_add_threads(info, dtc, DMA_PQ);
915 		thread_count += cnt > 0 ? cnt : 0;
916 	}
917 
918 	pr_info("Started %u threads using %s\n",
919 		thread_count, dma_chan_name(chan));
920 
921 	list_add_tail(&dtc->node, &info->channels);
922 	info->nr_channels++;
923 
924 	return 0;
925 }
926 
927 static bool filter(struct dma_chan *chan, void *param)
928 {
929 	struct dmatest_params *params = param;
930 
931 	if (!dmatest_match_channel(params, chan) ||
932 	    !dmatest_match_device(params, chan->device))
933 		return false;
934 	else
935 		return true;
936 }
937 
938 static void request_channels(struct dmatest_info *info,
939 			     enum dma_transaction_type type)
940 {
941 	dma_cap_mask_t mask;
942 
943 	dma_cap_zero(mask);
944 	dma_cap_set(type, mask);
945 	for (;;) {
946 		struct dmatest_params *params = &info->params;
947 		struct dma_chan *chan;
948 
949 		chan = dma_request_channel(mask, filter, params);
950 		if (chan) {
951 			if (dmatest_add_channel(info, chan)) {
952 				dma_release_channel(chan);
953 				break; /* add_channel failed, punt */
954 			}
955 		} else
956 			break; /* no more channels available */
957 		if (params->max_channels &&
958 		    info->nr_channels >= params->max_channels)
959 			break; /* we have all we need */
960 	}
961 }
962 
963 static void run_threaded_test(struct dmatest_info *info)
964 {
965 	struct dmatest_params *params = &info->params;
966 
967 	/* Copy test parameters */
968 	params->buf_size = test_buf_size;
969 	strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
970 	strlcpy(params->device, strim(test_device), sizeof(params->device));
971 	params->threads_per_chan = threads_per_chan;
972 	params->max_channels = max_channels;
973 	params->iterations = iterations;
974 	params->xor_sources = xor_sources;
975 	params->pq_sources = pq_sources;
976 	params->timeout = timeout;
977 	params->noverify = noverify;
978 
979 	request_channels(info, DMA_MEMCPY);
980 	request_channels(info, DMA_MEMSET);
981 	request_channels(info, DMA_XOR);
982 	request_channels(info, DMA_PQ);
983 }
984 
985 static void stop_threaded_test(struct dmatest_info *info)
986 {
987 	struct dmatest_chan *dtc, *_dtc;
988 	struct dma_chan *chan;
989 
990 	list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
991 		list_del(&dtc->node);
992 		chan = dtc->chan;
993 		dmatest_cleanup_channel(dtc);
994 		pr_debug("dropped channel %s\n", dma_chan_name(chan));
995 		dma_release_channel(chan);
996 	}
997 
998 	info->nr_channels = 0;
999 }
1000 
1001 static void restart_threaded_test(struct dmatest_info *info, bool run)
1002 {
1003 	/* we might be called early to set run=, defer running until all
1004 	 * parameters have been evaluated
1005 	 */
1006 	if (!info->did_init)
1007 		return;
1008 
1009 	/* Stop any running test first */
1010 	stop_threaded_test(info);
1011 
1012 	/* Run test with new parameters */
1013 	run_threaded_test(info);
1014 }
1015 
1016 static int dmatest_run_get(char *val, const struct kernel_param *kp)
1017 {
1018 	struct dmatest_info *info = &test_info;
1019 
1020 	mutex_lock(&info->lock);
1021 	if (is_threaded_test_run(info)) {
1022 		dmatest_run = true;
1023 	} else {
1024 		stop_threaded_test(info);
1025 		dmatest_run = false;
1026 	}
1027 	mutex_unlock(&info->lock);
1028 
1029 	return param_get_bool(val, kp);
1030 }
1031 
1032 static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1033 {
1034 	struct dmatest_info *info = &test_info;
1035 	int ret;
1036 
1037 	mutex_lock(&info->lock);
1038 	ret = param_set_bool(val, kp);
1039 	if (ret) {
1040 		mutex_unlock(&info->lock);
1041 		return ret;
1042 	}
1043 
1044 	if (is_threaded_test_run(info))
1045 		ret = -EBUSY;
1046 	else if (dmatest_run)
1047 		restart_threaded_test(info, dmatest_run);
1048 
1049 	mutex_unlock(&info->lock);
1050 
1051 	return ret;
1052 }
1053 
1054 static int __init dmatest_init(void)
1055 {
1056 	struct dmatest_info *info = &test_info;
1057 	struct dmatest_params *params = &info->params;
1058 
1059 	if (dmatest_run) {
1060 		mutex_lock(&info->lock);
1061 		run_threaded_test(info);
1062 		mutex_unlock(&info->lock);
1063 	}
1064 
1065 	if (params->iterations && wait)
1066 		wait_event(thread_wait, !is_threaded_test_run(info));
1067 
1068 	/* module parameters are stable, inittime tests are started,
1069 	 * let userspace take over 'run' control
1070 	 */
1071 	info->did_init = true;
1072 
1073 	return 0;
1074 }
1075 /* when compiled-in wait for drivers to load first */
1076 late_initcall(dmatest_init);
1077 
1078 static void __exit dmatest_exit(void)
1079 {
1080 	struct dmatest_info *info = &test_info;
1081 
1082 	mutex_lock(&info->lock);
1083 	stop_threaded_test(info);
1084 	mutex_unlock(&info->lock);
1085 }
1086 module_exit(dmatest_exit);
1087 
1088 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1089 MODULE_LICENSE("GPL v2");
1090