1 /* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/delay.h> 11 #include <linux/dmaengine.h> 12 #include <linux/init.h> 13 #include <linux/kthread.h> 14 #include <linux/module.h> 15 #include <linux/moduleparam.h> 16 #include <linux/random.h> 17 #include <linux/slab.h> 18 #include <linux/wait.h> 19 20 static unsigned int test_buf_size = 16384; 21 module_param(test_buf_size, uint, S_IRUGO); 22 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 23 24 static char test_channel[20]; 25 module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO); 26 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 27 28 static char test_device[20]; 29 module_param_string(device, test_device, sizeof(test_device), S_IRUGO); 30 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 31 32 static unsigned int threads_per_chan = 1; 33 module_param(threads_per_chan, uint, S_IRUGO); 34 MODULE_PARM_DESC(threads_per_chan, 35 "Number of threads to start per channel (default: 1)"); 36 37 static unsigned int max_channels; 38 module_param(max_channels, uint, S_IRUGO); 39 MODULE_PARM_DESC(max_channels, 40 "Maximum number of channels to use (default: all)"); 41 42 static unsigned int iterations; 43 module_param(iterations, uint, S_IRUGO); 44 MODULE_PARM_DESC(iterations, 45 "Iterations before stopping test (default: infinite)"); 46 47 static unsigned int xor_sources = 3; 48 module_param(xor_sources, uint, S_IRUGO); 49 MODULE_PARM_DESC(xor_sources, 50 "Number of xor source buffers (default: 3)"); 51 52 static unsigned int pq_sources = 3; 53 module_param(pq_sources, uint, S_IRUGO); 54 MODULE_PARM_DESC(pq_sources, 55 "Number of p+q source buffers (default: 3)"); 56 57 static int timeout = 3000; 58 module_param(timeout, uint, S_IRUGO); 59 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 60 "Pass -1 for infinite timeout"); 61 62 /* 63 * Initialization patterns. All bytes in the source buffer has bit 7 64 * set, all bytes in the destination buffer has bit 7 cleared. 65 * 66 * Bit 6 is set for all bytes which are to be copied by the DMA 67 * engine. Bit 5 is set for all bytes which are to be overwritten by 68 * the DMA engine. 69 * 70 * The remaining bits are the inverse of a counter which increments by 71 * one for each byte address. 72 */ 73 #define PATTERN_SRC 0x80 74 #define PATTERN_DST 0x00 75 #define PATTERN_COPY 0x40 76 #define PATTERN_OVERWRITE 0x20 77 #define PATTERN_COUNT_MASK 0x1f 78 79 struct dmatest_thread { 80 struct list_head node; 81 struct task_struct *task; 82 struct dma_chan *chan; 83 u8 **srcs; 84 u8 **dsts; 85 enum dma_transaction_type type; 86 }; 87 88 struct dmatest_chan { 89 struct list_head node; 90 struct dma_chan *chan; 91 struct list_head threads; 92 }; 93 94 /* 95 * These are protected by dma_list_mutex since they're only used by 96 * the DMA filter function callback 97 */ 98 static LIST_HEAD(dmatest_channels); 99 static unsigned int nr_channels; 100 101 static bool dmatest_match_channel(struct dma_chan *chan) 102 { 103 if (test_channel[0] == '\0') 104 return true; 105 return strcmp(dma_chan_name(chan), test_channel) == 0; 106 } 107 108 static bool dmatest_match_device(struct dma_device *device) 109 { 110 if (test_device[0] == '\0') 111 return true; 112 return strcmp(dev_name(device->dev), test_device) == 0; 113 } 114 115 static unsigned long dmatest_random(void) 116 { 117 unsigned long buf; 118 119 get_random_bytes(&buf, sizeof(buf)); 120 return buf; 121 } 122 123 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len) 124 { 125 unsigned int i; 126 u8 *buf; 127 128 for (; (buf = *bufs); bufs++) { 129 for (i = 0; i < start; i++) 130 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 131 for ( ; i < start + len; i++) 132 buf[i] = PATTERN_SRC | PATTERN_COPY 133 | (~i & PATTERN_COUNT_MASK); 134 for ( ; i < test_buf_size; i++) 135 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 136 buf++; 137 } 138 } 139 140 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len) 141 { 142 unsigned int i; 143 u8 *buf; 144 145 for (; (buf = *bufs); bufs++) { 146 for (i = 0; i < start; i++) 147 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 148 for ( ; i < start + len; i++) 149 buf[i] = PATTERN_DST | PATTERN_OVERWRITE 150 | (~i & PATTERN_COUNT_MASK); 151 for ( ; i < test_buf_size; i++) 152 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 153 } 154 } 155 156 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 157 unsigned int counter, bool is_srcbuf) 158 { 159 u8 diff = actual ^ pattern; 160 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 161 const char *thread_name = current->comm; 162 163 if (is_srcbuf) 164 pr_warning("%s: srcbuf[0x%x] overwritten!" 165 " Expected %02x, got %02x\n", 166 thread_name, index, expected, actual); 167 else if ((pattern & PATTERN_COPY) 168 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 169 pr_warning("%s: dstbuf[0x%x] not copied!" 170 " Expected %02x, got %02x\n", 171 thread_name, index, expected, actual); 172 else if (diff & PATTERN_SRC) 173 pr_warning("%s: dstbuf[0x%x] was copied!" 174 " Expected %02x, got %02x\n", 175 thread_name, index, expected, actual); 176 else 177 pr_warning("%s: dstbuf[0x%x] mismatch!" 178 " Expected %02x, got %02x\n", 179 thread_name, index, expected, actual); 180 } 181 182 static unsigned int dmatest_verify(u8 **bufs, unsigned int start, 183 unsigned int end, unsigned int counter, u8 pattern, 184 bool is_srcbuf) 185 { 186 unsigned int i; 187 unsigned int error_count = 0; 188 u8 actual; 189 u8 expected; 190 u8 *buf; 191 unsigned int counter_orig = counter; 192 193 for (; (buf = *bufs); bufs++) { 194 counter = counter_orig; 195 for (i = start; i < end; i++) { 196 actual = buf[i]; 197 expected = pattern | (~counter & PATTERN_COUNT_MASK); 198 if (actual != expected) { 199 if (error_count < 32) 200 dmatest_mismatch(actual, pattern, i, 201 counter, is_srcbuf); 202 error_count++; 203 } 204 counter++; 205 } 206 } 207 208 if (error_count > 32) 209 pr_warning("%s: %u errors suppressed\n", 210 current->comm, error_count - 32); 211 212 return error_count; 213 } 214 215 static void dmatest_callback(void *completion) 216 { 217 complete(completion); 218 } 219 220 /* 221 * This function repeatedly tests DMA transfers of various lengths and 222 * offsets for a given operation type until it is told to exit by 223 * kthread_stop(). There may be multiple threads running this function 224 * in parallel for a single channel, and there may be multiple channels 225 * being tested in parallel. 226 * 227 * Before each test, the source and destination buffer is initialized 228 * with a known pattern. This pattern is different depending on 229 * whether it's in an area which is supposed to be copied or 230 * overwritten, and different in the source and destination buffers. 231 * So if the DMA engine doesn't copy exactly what we tell it to copy, 232 * we'll notice. 233 */ 234 static int dmatest_func(void *data) 235 { 236 struct dmatest_thread *thread = data; 237 struct dma_chan *chan; 238 const char *thread_name; 239 unsigned int src_off, dst_off, len; 240 unsigned int error_count; 241 unsigned int failed_tests = 0; 242 unsigned int total_tests = 0; 243 dma_cookie_t cookie; 244 enum dma_status status; 245 enum dma_ctrl_flags flags; 246 u8 pq_coefs[pq_sources + 1]; 247 int ret; 248 int src_cnt; 249 int dst_cnt; 250 int i; 251 252 thread_name = current->comm; 253 254 ret = -ENOMEM; 255 256 smp_rmb(); 257 chan = thread->chan; 258 if (thread->type == DMA_MEMCPY) 259 src_cnt = dst_cnt = 1; 260 else if (thread->type == DMA_XOR) { 261 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */ 262 dst_cnt = 1; 263 } else if (thread->type == DMA_PQ) { 264 src_cnt = pq_sources | 1; /* force odd to ensure dst = src */ 265 dst_cnt = 2; 266 for (i = 0; i < src_cnt; i++) 267 pq_coefs[i] = 1; 268 } else 269 goto err_srcs; 270 271 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 272 if (!thread->srcs) 273 goto err_srcs; 274 for (i = 0; i < src_cnt; i++) { 275 thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL); 276 if (!thread->srcs[i]) 277 goto err_srcbuf; 278 } 279 thread->srcs[i] = NULL; 280 281 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 282 if (!thread->dsts) 283 goto err_dsts; 284 for (i = 0; i < dst_cnt; i++) { 285 thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL); 286 if (!thread->dsts[i]) 287 goto err_dstbuf; 288 } 289 thread->dsts[i] = NULL; 290 291 set_user_nice(current, 10); 292 293 /* 294 * src buffers are freed by the DMAEngine code with dma_unmap_single() 295 * dst buffers are freed by ourselves below 296 */ 297 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT 298 | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE; 299 300 while (!kthread_should_stop() 301 && !(iterations && total_tests >= iterations)) { 302 struct dma_device *dev = chan->device; 303 struct dma_async_tx_descriptor *tx = NULL; 304 dma_addr_t dma_srcs[src_cnt]; 305 dma_addr_t dma_dsts[dst_cnt]; 306 struct completion cmp; 307 unsigned long tmo = msecs_to_jiffies(timeout); 308 u8 align = 0; 309 310 total_tests++; 311 312 /* honor alignment restrictions */ 313 if (thread->type == DMA_MEMCPY) 314 align = dev->copy_align; 315 else if (thread->type == DMA_XOR) 316 align = dev->xor_align; 317 else if (thread->type == DMA_PQ) 318 align = dev->pq_align; 319 320 if (1 << align > test_buf_size) { 321 pr_err("%u-byte buffer too small for %d-byte alignment\n", 322 test_buf_size, 1 << align); 323 break; 324 } 325 326 len = dmatest_random() % test_buf_size + 1; 327 len = (len >> align) << align; 328 if (!len) 329 len = 1 << align; 330 src_off = dmatest_random() % (test_buf_size - len + 1); 331 dst_off = dmatest_random() % (test_buf_size - len + 1); 332 333 src_off = (src_off >> align) << align; 334 dst_off = (dst_off >> align) << align; 335 336 dmatest_init_srcs(thread->srcs, src_off, len); 337 dmatest_init_dsts(thread->dsts, dst_off, len); 338 339 for (i = 0; i < src_cnt; i++) { 340 u8 *buf = thread->srcs[i] + src_off; 341 342 dma_srcs[i] = dma_map_single(dev->dev, buf, len, 343 DMA_TO_DEVICE); 344 } 345 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 346 for (i = 0; i < dst_cnt; i++) { 347 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 348 test_buf_size, 349 DMA_BIDIRECTIONAL); 350 } 351 352 353 if (thread->type == DMA_MEMCPY) 354 tx = dev->device_prep_dma_memcpy(chan, 355 dma_dsts[0] + dst_off, 356 dma_srcs[0], len, 357 flags); 358 else if (thread->type == DMA_XOR) 359 tx = dev->device_prep_dma_xor(chan, 360 dma_dsts[0] + dst_off, 361 dma_srcs, src_cnt, 362 len, flags); 363 else if (thread->type == DMA_PQ) { 364 dma_addr_t dma_pq[dst_cnt]; 365 366 for (i = 0; i < dst_cnt; i++) 367 dma_pq[i] = dma_dsts[i] + dst_off; 368 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, 369 src_cnt, pq_coefs, 370 len, flags); 371 } 372 373 if (!tx) { 374 for (i = 0; i < src_cnt; i++) 375 dma_unmap_single(dev->dev, dma_srcs[i], len, 376 DMA_TO_DEVICE); 377 for (i = 0; i < dst_cnt; i++) 378 dma_unmap_single(dev->dev, dma_dsts[i], 379 test_buf_size, 380 DMA_BIDIRECTIONAL); 381 pr_warning("%s: #%u: prep error with src_off=0x%x " 382 "dst_off=0x%x len=0x%x\n", 383 thread_name, total_tests - 1, 384 src_off, dst_off, len); 385 msleep(100); 386 failed_tests++; 387 continue; 388 } 389 390 init_completion(&cmp); 391 tx->callback = dmatest_callback; 392 tx->callback_param = &cmp; 393 cookie = tx->tx_submit(tx); 394 395 if (dma_submit_error(cookie)) { 396 pr_warning("%s: #%u: submit error %d with src_off=0x%x " 397 "dst_off=0x%x len=0x%x\n", 398 thread_name, total_tests - 1, cookie, 399 src_off, dst_off, len); 400 msleep(100); 401 failed_tests++; 402 continue; 403 } 404 dma_async_issue_pending(chan); 405 406 tmo = wait_for_completion_timeout(&cmp, tmo); 407 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 408 409 if (tmo == 0) { 410 pr_warning("%s: #%u: test timed out\n", 411 thread_name, total_tests - 1); 412 failed_tests++; 413 continue; 414 } else if (status != DMA_SUCCESS) { 415 pr_warning("%s: #%u: got completion callback," 416 " but status is \'%s\'\n", 417 thread_name, total_tests - 1, 418 status == DMA_ERROR ? "error" : "in progress"); 419 failed_tests++; 420 continue; 421 } 422 423 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 424 for (i = 0; i < dst_cnt; i++) 425 dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size, 426 DMA_BIDIRECTIONAL); 427 428 error_count = 0; 429 430 pr_debug("%s: verifying source buffer...\n", thread_name); 431 error_count += dmatest_verify(thread->srcs, 0, src_off, 432 0, PATTERN_SRC, true); 433 error_count += dmatest_verify(thread->srcs, src_off, 434 src_off + len, src_off, 435 PATTERN_SRC | PATTERN_COPY, true); 436 error_count += dmatest_verify(thread->srcs, src_off + len, 437 test_buf_size, src_off + len, 438 PATTERN_SRC, true); 439 440 pr_debug("%s: verifying dest buffer...\n", 441 thread->task->comm); 442 error_count += dmatest_verify(thread->dsts, 0, dst_off, 443 0, PATTERN_DST, false); 444 error_count += dmatest_verify(thread->dsts, dst_off, 445 dst_off + len, src_off, 446 PATTERN_SRC | PATTERN_COPY, false); 447 error_count += dmatest_verify(thread->dsts, dst_off + len, 448 test_buf_size, dst_off + len, 449 PATTERN_DST, false); 450 451 if (error_count) { 452 pr_warning("%s: #%u: %u errors with " 453 "src_off=0x%x dst_off=0x%x len=0x%x\n", 454 thread_name, total_tests - 1, error_count, 455 src_off, dst_off, len); 456 failed_tests++; 457 } else { 458 pr_debug("%s: #%u: No errors with " 459 "src_off=0x%x dst_off=0x%x len=0x%x\n", 460 thread_name, total_tests - 1, 461 src_off, dst_off, len); 462 } 463 } 464 465 ret = 0; 466 for (i = 0; thread->dsts[i]; i++) 467 kfree(thread->dsts[i]); 468 err_dstbuf: 469 kfree(thread->dsts); 470 err_dsts: 471 for (i = 0; thread->srcs[i]; i++) 472 kfree(thread->srcs[i]); 473 err_srcbuf: 474 kfree(thread->srcs); 475 err_srcs: 476 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 477 thread_name, total_tests, failed_tests, ret); 478 479 if (iterations > 0) 480 while (!kthread_should_stop()) { 481 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); 482 interruptible_sleep_on(&wait_dmatest_exit); 483 } 484 485 return ret; 486 } 487 488 static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 489 { 490 struct dmatest_thread *thread; 491 struct dmatest_thread *_thread; 492 int ret; 493 494 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 495 ret = kthread_stop(thread->task); 496 pr_debug("dmatest: thread %s exited with status %d\n", 497 thread->task->comm, ret); 498 list_del(&thread->node); 499 kfree(thread); 500 } 501 kfree(dtc); 502 } 503 504 static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type) 505 { 506 struct dmatest_thread *thread; 507 struct dma_chan *chan = dtc->chan; 508 char *op; 509 unsigned int i; 510 511 if (type == DMA_MEMCPY) 512 op = "copy"; 513 else if (type == DMA_XOR) 514 op = "xor"; 515 else if (type == DMA_PQ) 516 op = "pq"; 517 else 518 return -EINVAL; 519 520 for (i = 0; i < threads_per_chan; i++) { 521 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 522 if (!thread) { 523 pr_warning("dmatest: No memory for %s-%s%u\n", 524 dma_chan_name(chan), op, i); 525 526 break; 527 } 528 thread->chan = dtc->chan; 529 thread->type = type; 530 smp_wmb(); 531 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 532 dma_chan_name(chan), op, i); 533 if (IS_ERR(thread->task)) { 534 pr_warning("dmatest: Failed to run thread %s-%s%u\n", 535 dma_chan_name(chan), op, i); 536 kfree(thread); 537 break; 538 } 539 540 /* srcbuf and dstbuf are allocated by the thread itself */ 541 542 list_add_tail(&thread->node, &dtc->threads); 543 } 544 545 return i; 546 } 547 548 static int dmatest_add_channel(struct dma_chan *chan) 549 { 550 struct dmatest_chan *dtc; 551 struct dma_device *dma_dev = chan->device; 552 unsigned int thread_count = 0; 553 int cnt; 554 555 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 556 if (!dtc) { 557 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 558 return -ENOMEM; 559 } 560 561 dtc->chan = chan; 562 INIT_LIST_HEAD(&dtc->threads); 563 564 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 565 cnt = dmatest_add_threads(dtc, DMA_MEMCPY); 566 thread_count += cnt > 0 ? cnt : 0; 567 } 568 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 569 cnt = dmatest_add_threads(dtc, DMA_XOR); 570 thread_count += cnt > 0 ? cnt : 0; 571 } 572 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 573 cnt = dmatest_add_threads(dtc, DMA_PQ); 574 thread_count += cnt > 0 ?: 0; 575 } 576 577 pr_info("dmatest: Started %u threads using %s\n", 578 thread_count, dma_chan_name(chan)); 579 580 list_add_tail(&dtc->node, &dmatest_channels); 581 nr_channels++; 582 583 return 0; 584 } 585 586 static bool filter(struct dma_chan *chan, void *param) 587 { 588 if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device)) 589 return false; 590 else 591 return true; 592 } 593 594 static int __init dmatest_init(void) 595 { 596 dma_cap_mask_t mask; 597 struct dma_chan *chan; 598 int err = 0; 599 600 dma_cap_zero(mask); 601 dma_cap_set(DMA_MEMCPY, mask); 602 for (;;) { 603 chan = dma_request_channel(mask, filter, NULL); 604 if (chan) { 605 err = dmatest_add_channel(chan); 606 if (err) { 607 dma_release_channel(chan); 608 break; /* add_channel failed, punt */ 609 } 610 } else 611 break; /* no more channels available */ 612 if (max_channels && nr_channels >= max_channels) 613 break; /* we have all we need */ 614 } 615 616 return err; 617 } 618 /* when compiled-in wait for drivers to load first */ 619 late_initcall(dmatest_init); 620 621 static void __exit dmatest_exit(void) 622 { 623 struct dmatest_chan *dtc, *_dtc; 624 struct dma_chan *chan; 625 626 list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) { 627 list_del(&dtc->node); 628 chan = dtc->chan; 629 dmatest_cleanup_channel(dtc); 630 pr_debug("dmatest: dropped channel %s\n", 631 dma_chan_name(chan)); 632 dma_release_channel(chan); 633 } 634 } 635 module_exit(dmatest_exit); 636 637 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 638 MODULE_LICENSE("GPL v2"); 639