1 /* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/dmaengine.h> 13 #include <linux/freezer.h> 14 #include <linux/init.h> 15 #include <linux/kthread.h> 16 #include <linux/module.h> 17 #include <linux/moduleparam.h> 18 #include <linux/random.h> 19 #include <linux/slab.h> 20 #include <linux/wait.h> 21 22 static unsigned int test_buf_size = 16384; 23 module_param(test_buf_size, uint, S_IRUGO); 24 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 25 26 static char test_channel[20]; 27 module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO); 28 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 29 30 static char test_device[20]; 31 module_param_string(device, test_device, sizeof(test_device), S_IRUGO); 32 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 33 34 static unsigned int threads_per_chan = 1; 35 module_param(threads_per_chan, uint, S_IRUGO); 36 MODULE_PARM_DESC(threads_per_chan, 37 "Number of threads to start per channel (default: 1)"); 38 39 static unsigned int max_channels; 40 module_param(max_channels, uint, S_IRUGO); 41 MODULE_PARM_DESC(max_channels, 42 "Maximum number of channels to use (default: all)"); 43 44 static unsigned int iterations; 45 module_param(iterations, uint, S_IRUGO); 46 MODULE_PARM_DESC(iterations, 47 "Iterations before stopping test (default: infinite)"); 48 49 static unsigned int xor_sources = 3; 50 module_param(xor_sources, uint, S_IRUGO); 51 MODULE_PARM_DESC(xor_sources, 52 "Number of xor source buffers (default: 3)"); 53 54 static unsigned int pq_sources = 3; 55 module_param(pq_sources, uint, S_IRUGO); 56 MODULE_PARM_DESC(pq_sources, 57 "Number of p+q source buffers (default: 3)"); 58 59 static int timeout = 3000; 60 module_param(timeout, uint, S_IRUGO); 61 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 62 "Pass -1 for infinite timeout"); 63 64 /* 65 * Initialization patterns. All bytes in the source buffer has bit 7 66 * set, all bytes in the destination buffer has bit 7 cleared. 67 * 68 * Bit 6 is set for all bytes which are to be copied by the DMA 69 * engine. Bit 5 is set for all bytes which are to be overwritten by 70 * the DMA engine. 71 * 72 * The remaining bits are the inverse of a counter which increments by 73 * one for each byte address. 74 */ 75 #define PATTERN_SRC 0x80 76 #define PATTERN_DST 0x00 77 #define PATTERN_COPY 0x40 78 #define PATTERN_OVERWRITE 0x20 79 #define PATTERN_COUNT_MASK 0x1f 80 81 struct dmatest_thread { 82 struct list_head node; 83 struct task_struct *task; 84 struct dma_chan *chan; 85 u8 **srcs; 86 u8 **dsts; 87 enum dma_transaction_type type; 88 }; 89 90 struct dmatest_chan { 91 struct list_head node; 92 struct dma_chan *chan; 93 struct list_head threads; 94 }; 95 96 /* 97 * These are protected by dma_list_mutex since they're only used by 98 * the DMA filter function callback 99 */ 100 static LIST_HEAD(dmatest_channels); 101 static unsigned int nr_channels; 102 103 static bool dmatest_match_channel(struct dma_chan *chan) 104 { 105 if (test_channel[0] == '\0') 106 return true; 107 return strcmp(dma_chan_name(chan), test_channel) == 0; 108 } 109 110 static bool dmatest_match_device(struct dma_device *device) 111 { 112 if (test_device[0] == '\0') 113 return true; 114 return strcmp(dev_name(device->dev), test_device) == 0; 115 } 116 117 static unsigned long dmatest_random(void) 118 { 119 unsigned long buf; 120 121 get_random_bytes(&buf, sizeof(buf)); 122 return buf; 123 } 124 125 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len) 126 { 127 unsigned int i; 128 u8 *buf; 129 130 for (; (buf = *bufs); bufs++) { 131 for (i = 0; i < start; i++) 132 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 133 for ( ; i < start + len; i++) 134 buf[i] = PATTERN_SRC | PATTERN_COPY 135 | (~i & PATTERN_COUNT_MASK); 136 for ( ; i < test_buf_size; i++) 137 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 138 buf++; 139 } 140 } 141 142 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len) 143 { 144 unsigned int i; 145 u8 *buf; 146 147 for (; (buf = *bufs); bufs++) { 148 for (i = 0; i < start; i++) 149 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 150 for ( ; i < start + len; i++) 151 buf[i] = PATTERN_DST | PATTERN_OVERWRITE 152 | (~i & PATTERN_COUNT_MASK); 153 for ( ; i < test_buf_size; i++) 154 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 155 } 156 } 157 158 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 159 unsigned int counter, bool is_srcbuf) 160 { 161 u8 diff = actual ^ pattern; 162 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 163 const char *thread_name = current->comm; 164 165 if (is_srcbuf) 166 pr_warning("%s: srcbuf[0x%x] overwritten!" 167 " Expected %02x, got %02x\n", 168 thread_name, index, expected, actual); 169 else if ((pattern & PATTERN_COPY) 170 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 171 pr_warning("%s: dstbuf[0x%x] not copied!" 172 " Expected %02x, got %02x\n", 173 thread_name, index, expected, actual); 174 else if (diff & PATTERN_SRC) 175 pr_warning("%s: dstbuf[0x%x] was copied!" 176 " Expected %02x, got %02x\n", 177 thread_name, index, expected, actual); 178 else 179 pr_warning("%s: dstbuf[0x%x] mismatch!" 180 " Expected %02x, got %02x\n", 181 thread_name, index, expected, actual); 182 } 183 184 static unsigned int dmatest_verify(u8 **bufs, unsigned int start, 185 unsigned int end, unsigned int counter, u8 pattern, 186 bool is_srcbuf) 187 { 188 unsigned int i; 189 unsigned int error_count = 0; 190 u8 actual; 191 u8 expected; 192 u8 *buf; 193 unsigned int counter_orig = counter; 194 195 for (; (buf = *bufs); bufs++) { 196 counter = counter_orig; 197 for (i = start; i < end; i++) { 198 actual = buf[i]; 199 expected = pattern | (~counter & PATTERN_COUNT_MASK); 200 if (actual != expected) { 201 if (error_count < 32) 202 dmatest_mismatch(actual, pattern, i, 203 counter, is_srcbuf); 204 error_count++; 205 } 206 counter++; 207 } 208 } 209 210 if (error_count > 32) 211 pr_warning("%s: %u errors suppressed\n", 212 current->comm, error_count - 32); 213 214 return error_count; 215 } 216 217 /* poor man's completion - we want to use wait_event_freezable() on it */ 218 struct dmatest_done { 219 bool done; 220 wait_queue_head_t *wait; 221 }; 222 223 static void dmatest_callback(void *arg) 224 { 225 struct dmatest_done *done = arg; 226 227 done->done = true; 228 wake_up_all(done->wait); 229 } 230 231 /* 232 * This function repeatedly tests DMA transfers of various lengths and 233 * offsets for a given operation type until it is told to exit by 234 * kthread_stop(). There may be multiple threads running this function 235 * in parallel for a single channel, and there may be multiple channels 236 * being tested in parallel. 237 * 238 * Before each test, the source and destination buffer is initialized 239 * with a known pattern. This pattern is different depending on 240 * whether it's in an area which is supposed to be copied or 241 * overwritten, and different in the source and destination buffers. 242 * So if the DMA engine doesn't copy exactly what we tell it to copy, 243 * we'll notice. 244 */ 245 static int dmatest_func(void *data) 246 { 247 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait); 248 struct dmatest_thread *thread = data; 249 struct dmatest_done done = { .wait = &done_wait }; 250 struct dma_chan *chan; 251 const char *thread_name; 252 unsigned int src_off, dst_off, len; 253 unsigned int error_count; 254 unsigned int failed_tests = 0; 255 unsigned int total_tests = 0; 256 dma_cookie_t cookie; 257 enum dma_status status; 258 enum dma_ctrl_flags flags; 259 u8 pq_coefs[pq_sources + 1]; 260 int ret; 261 int src_cnt; 262 int dst_cnt; 263 int i; 264 265 thread_name = current->comm; 266 set_freezable(); 267 268 ret = -ENOMEM; 269 270 smp_rmb(); 271 chan = thread->chan; 272 if (thread->type == DMA_MEMCPY) 273 src_cnt = dst_cnt = 1; 274 else if (thread->type == DMA_XOR) { 275 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */ 276 dst_cnt = 1; 277 } else if (thread->type == DMA_PQ) { 278 src_cnt = pq_sources | 1; /* force odd to ensure dst = src */ 279 dst_cnt = 2; 280 for (i = 0; i < src_cnt; i++) 281 pq_coefs[i] = 1; 282 } else 283 goto err_srcs; 284 285 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 286 if (!thread->srcs) 287 goto err_srcs; 288 for (i = 0; i < src_cnt; i++) { 289 thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL); 290 if (!thread->srcs[i]) 291 goto err_srcbuf; 292 } 293 thread->srcs[i] = NULL; 294 295 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 296 if (!thread->dsts) 297 goto err_dsts; 298 for (i = 0; i < dst_cnt; i++) { 299 thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL); 300 if (!thread->dsts[i]) 301 goto err_dstbuf; 302 } 303 thread->dsts[i] = NULL; 304 305 set_user_nice(current, 10); 306 307 /* 308 * src buffers are freed by the DMAEngine code with dma_unmap_single() 309 * dst buffers are freed by ourselves below 310 */ 311 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT 312 | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE; 313 314 while (!kthread_should_stop() 315 && !(iterations && total_tests >= iterations)) { 316 struct dma_device *dev = chan->device; 317 struct dma_async_tx_descriptor *tx = NULL; 318 dma_addr_t dma_srcs[src_cnt]; 319 dma_addr_t dma_dsts[dst_cnt]; 320 u8 align = 0; 321 322 total_tests++; 323 324 /* honor alignment restrictions */ 325 if (thread->type == DMA_MEMCPY) 326 align = dev->copy_align; 327 else if (thread->type == DMA_XOR) 328 align = dev->xor_align; 329 else if (thread->type == DMA_PQ) 330 align = dev->pq_align; 331 332 if (1 << align > test_buf_size) { 333 pr_err("%u-byte buffer too small for %d-byte alignment\n", 334 test_buf_size, 1 << align); 335 break; 336 } 337 338 len = dmatest_random() % test_buf_size + 1; 339 len = (len >> align) << align; 340 if (!len) 341 len = 1 << align; 342 src_off = dmatest_random() % (test_buf_size - len + 1); 343 dst_off = dmatest_random() % (test_buf_size - len + 1); 344 345 src_off = (src_off >> align) << align; 346 dst_off = (dst_off >> align) << align; 347 348 dmatest_init_srcs(thread->srcs, src_off, len); 349 dmatest_init_dsts(thread->dsts, dst_off, len); 350 351 for (i = 0; i < src_cnt; i++) { 352 u8 *buf = thread->srcs[i] + src_off; 353 354 dma_srcs[i] = dma_map_single(dev->dev, buf, len, 355 DMA_TO_DEVICE); 356 } 357 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 358 for (i = 0; i < dst_cnt; i++) { 359 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 360 test_buf_size, 361 DMA_BIDIRECTIONAL); 362 } 363 364 365 if (thread->type == DMA_MEMCPY) 366 tx = dev->device_prep_dma_memcpy(chan, 367 dma_dsts[0] + dst_off, 368 dma_srcs[0], len, 369 flags); 370 else if (thread->type == DMA_XOR) 371 tx = dev->device_prep_dma_xor(chan, 372 dma_dsts[0] + dst_off, 373 dma_srcs, src_cnt, 374 len, flags); 375 else if (thread->type == DMA_PQ) { 376 dma_addr_t dma_pq[dst_cnt]; 377 378 for (i = 0; i < dst_cnt; i++) 379 dma_pq[i] = dma_dsts[i] + dst_off; 380 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, 381 src_cnt, pq_coefs, 382 len, flags); 383 } 384 385 if (!tx) { 386 for (i = 0; i < src_cnt; i++) 387 dma_unmap_single(dev->dev, dma_srcs[i], len, 388 DMA_TO_DEVICE); 389 for (i = 0; i < dst_cnt; i++) 390 dma_unmap_single(dev->dev, dma_dsts[i], 391 test_buf_size, 392 DMA_BIDIRECTIONAL); 393 pr_warning("%s: #%u: prep error with src_off=0x%x " 394 "dst_off=0x%x len=0x%x\n", 395 thread_name, total_tests - 1, 396 src_off, dst_off, len); 397 msleep(100); 398 failed_tests++; 399 continue; 400 } 401 402 done.done = false; 403 tx->callback = dmatest_callback; 404 tx->callback_param = &done; 405 cookie = tx->tx_submit(tx); 406 407 if (dma_submit_error(cookie)) { 408 pr_warning("%s: #%u: submit error %d with src_off=0x%x " 409 "dst_off=0x%x len=0x%x\n", 410 thread_name, total_tests - 1, cookie, 411 src_off, dst_off, len); 412 msleep(100); 413 failed_tests++; 414 continue; 415 } 416 dma_async_issue_pending(chan); 417 418 wait_event_freezable_timeout(done_wait, done.done, 419 msecs_to_jiffies(timeout)); 420 421 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 422 423 if (!done.done) { 424 /* 425 * We're leaving the timed out dma operation with 426 * dangling pointer to done_wait. To make this 427 * correct, we'll need to allocate wait_done for 428 * each test iteration and perform "who's gonna 429 * free it this time?" dancing. For now, just 430 * leave it dangling. 431 */ 432 pr_warning("%s: #%u: test timed out\n", 433 thread_name, total_tests - 1); 434 failed_tests++; 435 continue; 436 } else if (status != DMA_SUCCESS) { 437 pr_warning("%s: #%u: got completion callback," 438 " but status is \'%s\'\n", 439 thread_name, total_tests - 1, 440 status == DMA_ERROR ? "error" : "in progress"); 441 failed_tests++; 442 continue; 443 } 444 445 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 446 for (i = 0; i < dst_cnt; i++) 447 dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size, 448 DMA_BIDIRECTIONAL); 449 450 error_count = 0; 451 452 pr_debug("%s: verifying source buffer...\n", thread_name); 453 error_count += dmatest_verify(thread->srcs, 0, src_off, 454 0, PATTERN_SRC, true); 455 error_count += dmatest_verify(thread->srcs, src_off, 456 src_off + len, src_off, 457 PATTERN_SRC | PATTERN_COPY, true); 458 error_count += dmatest_verify(thread->srcs, src_off + len, 459 test_buf_size, src_off + len, 460 PATTERN_SRC, true); 461 462 pr_debug("%s: verifying dest buffer...\n", 463 thread->task->comm); 464 error_count += dmatest_verify(thread->dsts, 0, dst_off, 465 0, PATTERN_DST, false); 466 error_count += dmatest_verify(thread->dsts, dst_off, 467 dst_off + len, src_off, 468 PATTERN_SRC | PATTERN_COPY, false); 469 error_count += dmatest_verify(thread->dsts, dst_off + len, 470 test_buf_size, dst_off + len, 471 PATTERN_DST, false); 472 473 if (error_count) { 474 pr_warning("%s: #%u: %u errors with " 475 "src_off=0x%x dst_off=0x%x len=0x%x\n", 476 thread_name, total_tests - 1, error_count, 477 src_off, dst_off, len); 478 failed_tests++; 479 } else { 480 pr_debug("%s: #%u: No errors with " 481 "src_off=0x%x dst_off=0x%x len=0x%x\n", 482 thread_name, total_tests - 1, 483 src_off, dst_off, len); 484 } 485 } 486 487 ret = 0; 488 for (i = 0; thread->dsts[i]; i++) 489 kfree(thread->dsts[i]); 490 err_dstbuf: 491 kfree(thread->dsts); 492 err_dsts: 493 for (i = 0; thread->srcs[i]; i++) 494 kfree(thread->srcs[i]); 495 err_srcbuf: 496 kfree(thread->srcs); 497 err_srcs: 498 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 499 thread_name, total_tests, failed_tests, ret); 500 501 /* terminate all transfers on specified channels */ 502 chan->device->device_control(chan, DMA_TERMINATE_ALL, 0); 503 if (iterations > 0) 504 while (!kthread_should_stop()) { 505 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); 506 interruptible_sleep_on(&wait_dmatest_exit); 507 } 508 509 return ret; 510 } 511 512 static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 513 { 514 struct dmatest_thread *thread; 515 struct dmatest_thread *_thread; 516 int ret; 517 518 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 519 ret = kthread_stop(thread->task); 520 pr_debug("dmatest: thread %s exited with status %d\n", 521 thread->task->comm, ret); 522 list_del(&thread->node); 523 kfree(thread); 524 } 525 526 /* terminate all transfers on specified channels */ 527 dtc->chan->device->device_control(dtc->chan, DMA_TERMINATE_ALL, 0); 528 529 kfree(dtc); 530 } 531 532 static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type) 533 { 534 struct dmatest_thread *thread; 535 struct dma_chan *chan = dtc->chan; 536 char *op; 537 unsigned int i; 538 539 if (type == DMA_MEMCPY) 540 op = "copy"; 541 else if (type == DMA_XOR) 542 op = "xor"; 543 else if (type == DMA_PQ) 544 op = "pq"; 545 else 546 return -EINVAL; 547 548 for (i = 0; i < threads_per_chan; i++) { 549 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 550 if (!thread) { 551 pr_warning("dmatest: No memory for %s-%s%u\n", 552 dma_chan_name(chan), op, i); 553 554 break; 555 } 556 thread->chan = dtc->chan; 557 thread->type = type; 558 smp_wmb(); 559 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 560 dma_chan_name(chan), op, i); 561 if (IS_ERR(thread->task)) { 562 pr_warning("dmatest: Failed to run thread %s-%s%u\n", 563 dma_chan_name(chan), op, i); 564 kfree(thread); 565 break; 566 } 567 568 /* srcbuf and dstbuf are allocated by the thread itself */ 569 570 list_add_tail(&thread->node, &dtc->threads); 571 } 572 573 return i; 574 } 575 576 static int dmatest_add_channel(struct dma_chan *chan) 577 { 578 struct dmatest_chan *dtc; 579 struct dma_device *dma_dev = chan->device; 580 unsigned int thread_count = 0; 581 int cnt; 582 583 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 584 if (!dtc) { 585 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 586 return -ENOMEM; 587 } 588 589 dtc->chan = chan; 590 INIT_LIST_HEAD(&dtc->threads); 591 592 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 593 cnt = dmatest_add_threads(dtc, DMA_MEMCPY); 594 thread_count += cnt > 0 ? cnt : 0; 595 } 596 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 597 cnt = dmatest_add_threads(dtc, DMA_XOR); 598 thread_count += cnt > 0 ? cnt : 0; 599 } 600 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 601 cnt = dmatest_add_threads(dtc, DMA_PQ); 602 thread_count += cnt > 0 ? cnt : 0; 603 } 604 605 pr_info("dmatest: Started %u threads using %s\n", 606 thread_count, dma_chan_name(chan)); 607 608 list_add_tail(&dtc->node, &dmatest_channels); 609 nr_channels++; 610 611 return 0; 612 } 613 614 static bool filter(struct dma_chan *chan, void *param) 615 { 616 if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device)) 617 return false; 618 else 619 return true; 620 } 621 622 static int __init dmatest_init(void) 623 { 624 dma_cap_mask_t mask; 625 struct dma_chan *chan; 626 int err = 0; 627 628 dma_cap_zero(mask); 629 dma_cap_set(DMA_MEMCPY, mask); 630 for (;;) { 631 chan = dma_request_channel(mask, filter, NULL); 632 if (chan) { 633 err = dmatest_add_channel(chan); 634 if (err) { 635 dma_release_channel(chan); 636 break; /* add_channel failed, punt */ 637 } 638 } else 639 break; /* no more channels available */ 640 if (max_channels && nr_channels >= max_channels) 641 break; /* we have all we need */ 642 } 643 644 return err; 645 } 646 /* when compiled-in wait for drivers to load first */ 647 late_initcall(dmatest_init); 648 649 static void __exit dmatest_exit(void) 650 { 651 struct dmatest_chan *dtc, *_dtc; 652 struct dma_chan *chan; 653 654 list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) { 655 list_del(&dtc->node); 656 chan = dtc->chan; 657 dmatest_cleanup_channel(dtc); 658 pr_debug("dmatest: dropped channel %s\n", 659 dma_chan_name(chan)); 660 dma_release_channel(chan); 661 } 662 } 663 module_exit(dmatest_exit); 664 665 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 666 MODULE_LICENSE("GPL v2"); 667