1 /* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * Copyright (C) 2013 Intel Corporation 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #include <linux/delay.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/dmaengine.h> 14 #include <linux/freezer.h> 15 #include <linux/init.h> 16 #include <linux/kthread.h> 17 #include <linux/module.h> 18 #include <linux/moduleparam.h> 19 #include <linux/random.h> 20 #include <linux/slab.h> 21 #include <linux/wait.h> 22 #include <linux/ctype.h> 23 #include <linux/debugfs.h> 24 #include <linux/uaccess.h> 25 #include <linux/seq_file.h> 26 27 static unsigned int test_buf_size = 16384; 28 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR); 29 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 30 31 static char test_channel[20]; 32 module_param_string(channel, test_channel, sizeof(test_channel), 33 S_IRUGO | S_IWUSR); 34 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 35 36 static char test_device[20]; 37 module_param_string(device, test_device, sizeof(test_device), 38 S_IRUGO | S_IWUSR); 39 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 40 41 static unsigned int threads_per_chan = 1; 42 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR); 43 MODULE_PARM_DESC(threads_per_chan, 44 "Number of threads to start per channel (default: 1)"); 45 46 static unsigned int max_channels; 47 module_param(max_channels, uint, S_IRUGO | S_IWUSR); 48 MODULE_PARM_DESC(max_channels, 49 "Maximum number of channels to use (default: all)"); 50 51 static unsigned int iterations; 52 module_param(iterations, uint, S_IRUGO | S_IWUSR); 53 MODULE_PARM_DESC(iterations, 54 "Iterations before stopping test (default: infinite)"); 55 56 static unsigned int xor_sources = 3; 57 module_param(xor_sources, uint, S_IRUGO | S_IWUSR); 58 MODULE_PARM_DESC(xor_sources, 59 "Number of xor source buffers (default: 3)"); 60 61 static unsigned int pq_sources = 3; 62 module_param(pq_sources, uint, S_IRUGO | S_IWUSR); 63 MODULE_PARM_DESC(pq_sources, 64 "Number of p+q source buffers (default: 3)"); 65 66 static int timeout = 3000; 67 module_param(timeout, uint, S_IRUGO | S_IWUSR); 68 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 69 "Pass -1 for infinite timeout"); 70 71 /* Maximum amount of mismatched bytes in buffer to print */ 72 #define MAX_ERROR_COUNT 32 73 74 /* 75 * Initialization patterns. All bytes in the source buffer has bit 7 76 * set, all bytes in the destination buffer has bit 7 cleared. 77 * 78 * Bit 6 is set for all bytes which are to be copied by the DMA 79 * engine. Bit 5 is set for all bytes which are to be overwritten by 80 * the DMA engine. 81 * 82 * The remaining bits are the inverse of a counter which increments by 83 * one for each byte address. 84 */ 85 #define PATTERN_SRC 0x80 86 #define PATTERN_DST 0x00 87 #define PATTERN_COPY 0x40 88 #define PATTERN_OVERWRITE 0x20 89 #define PATTERN_COUNT_MASK 0x1f 90 91 enum dmatest_error_type { 92 DMATEST_ET_OK, 93 DMATEST_ET_MAP_SRC, 94 DMATEST_ET_MAP_DST, 95 DMATEST_ET_PREP, 96 DMATEST_ET_SUBMIT, 97 DMATEST_ET_TIMEOUT, 98 DMATEST_ET_DMA_ERROR, 99 DMATEST_ET_DMA_IN_PROGRESS, 100 DMATEST_ET_VERIFY, 101 DMATEST_ET_VERIFY_BUF, 102 }; 103 104 struct dmatest_verify_buffer { 105 unsigned int index; 106 u8 expected; 107 u8 actual; 108 }; 109 110 struct dmatest_verify_result { 111 unsigned int error_count; 112 struct dmatest_verify_buffer data[MAX_ERROR_COUNT]; 113 u8 pattern; 114 bool is_srcbuf; 115 }; 116 117 struct dmatest_thread_result { 118 struct list_head node; 119 unsigned int n; 120 unsigned int src_off; 121 unsigned int dst_off; 122 unsigned int len; 123 enum dmatest_error_type type; 124 union { 125 unsigned long data; 126 dma_cookie_t cookie; 127 enum dma_status status; 128 int error; 129 struct dmatest_verify_result *vr; 130 }; 131 }; 132 133 struct dmatest_result { 134 struct list_head node; 135 char *name; 136 struct list_head results; 137 }; 138 139 struct dmatest_info; 140 141 struct dmatest_thread { 142 struct list_head node; 143 struct dmatest_info *info; 144 struct task_struct *task; 145 struct dma_chan *chan; 146 u8 **srcs; 147 u8 **dsts; 148 enum dma_transaction_type type; 149 bool done; 150 }; 151 152 struct dmatest_chan { 153 struct list_head node; 154 struct dma_chan *chan; 155 struct list_head threads; 156 }; 157 158 /** 159 * struct dmatest_params - test parameters. 160 * @buf_size: size of the memcpy test buffer 161 * @channel: bus ID of the channel to test 162 * @device: bus ID of the DMA Engine to test 163 * @threads_per_chan: number of threads to start per channel 164 * @max_channels: maximum number of channels to use 165 * @iterations: iterations before stopping test 166 * @xor_sources: number of xor source buffers 167 * @pq_sources: number of p+q source buffers 168 * @timeout: transfer timeout in msec, -1 for infinite timeout 169 */ 170 struct dmatest_params { 171 unsigned int buf_size; 172 char channel[20]; 173 char device[20]; 174 unsigned int threads_per_chan; 175 unsigned int max_channels; 176 unsigned int iterations; 177 unsigned int xor_sources; 178 unsigned int pq_sources; 179 int timeout; 180 }; 181 182 /** 183 * struct dmatest_info - test information. 184 * @params: test parameters 185 * @lock: access protection to the fields of this structure 186 */ 187 struct dmatest_info { 188 /* Test parameters */ 189 struct dmatest_params params; 190 191 /* Internal state */ 192 struct list_head channels; 193 unsigned int nr_channels; 194 struct mutex lock; 195 196 /* debugfs related stuff */ 197 struct dentry *root; 198 199 /* Test results */ 200 struct list_head results; 201 struct mutex results_lock; 202 }; 203 204 static struct dmatest_info test_info; 205 206 static bool dmatest_match_channel(struct dmatest_params *params, 207 struct dma_chan *chan) 208 { 209 if (params->channel[0] == '\0') 210 return true; 211 return strcmp(dma_chan_name(chan), params->channel) == 0; 212 } 213 214 static bool dmatest_match_device(struct dmatest_params *params, 215 struct dma_device *device) 216 { 217 if (params->device[0] == '\0') 218 return true; 219 return strcmp(dev_name(device->dev), params->device) == 0; 220 } 221 222 static unsigned long dmatest_random(void) 223 { 224 unsigned long buf; 225 226 get_random_bytes(&buf, sizeof(buf)); 227 return buf; 228 } 229 230 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len, 231 unsigned int buf_size) 232 { 233 unsigned int i; 234 u8 *buf; 235 236 for (; (buf = *bufs); bufs++) { 237 for (i = 0; i < start; i++) 238 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 239 for ( ; i < start + len; i++) 240 buf[i] = PATTERN_SRC | PATTERN_COPY 241 | (~i & PATTERN_COUNT_MASK); 242 for ( ; i < buf_size; i++) 243 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 244 buf++; 245 } 246 } 247 248 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len, 249 unsigned int buf_size) 250 { 251 unsigned int i; 252 u8 *buf; 253 254 for (; (buf = *bufs); bufs++) { 255 for (i = 0; i < start; i++) 256 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 257 for ( ; i < start + len; i++) 258 buf[i] = PATTERN_DST | PATTERN_OVERWRITE 259 | (~i & PATTERN_COUNT_MASK); 260 for ( ; i < buf_size; i++) 261 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 262 } 263 } 264 265 static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs, 266 unsigned int start, unsigned int end, unsigned int counter, 267 u8 pattern, bool is_srcbuf) 268 { 269 unsigned int i; 270 unsigned int error_count = 0; 271 u8 actual; 272 u8 expected; 273 u8 *buf; 274 unsigned int counter_orig = counter; 275 struct dmatest_verify_buffer *vb; 276 277 for (; (buf = *bufs); bufs++) { 278 counter = counter_orig; 279 for (i = start; i < end; i++) { 280 actual = buf[i]; 281 expected = pattern | (~counter & PATTERN_COUNT_MASK); 282 if (actual != expected) { 283 if (error_count < MAX_ERROR_COUNT && vr) { 284 vb = &vr->data[error_count]; 285 vb->index = i; 286 vb->expected = expected; 287 vb->actual = actual; 288 } 289 error_count++; 290 } 291 counter++; 292 } 293 } 294 295 if (error_count > MAX_ERROR_COUNT) 296 pr_warning("%s: %u errors suppressed\n", 297 current->comm, error_count - MAX_ERROR_COUNT); 298 299 return error_count; 300 } 301 302 /* poor man's completion - we want to use wait_event_freezable() on it */ 303 struct dmatest_done { 304 bool done; 305 wait_queue_head_t *wait; 306 }; 307 308 static void dmatest_callback(void *arg) 309 { 310 struct dmatest_done *done = arg; 311 312 done->done = true; 313 wake_up_all(done->wait); 314 } 315 316 static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len, 317 unsigned int count) 318 { 319 while (count--) 320 dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE); 321 } 322 323 static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len, 324 unsigned int count) 325 { 326 while (count--) 327 dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL); 328 } 329 330 static unsigned int min_odd(unsigned int x, unsigned int y) 331 { 332 unsigned int val = min(x, y); 333 334 return val % 2 ? val : val - 1; 335 } 336 337 static char *verify_result_get_one(struct dmatest_verify_result *vr, 338 unsigned int i) 339 { 340 struct dmatest_verify_buffer *vb = &vr->data[i]; 341 u8 diff = vb->actual ^ vr->pattern; 342 static char buf[512]; 343 char *msg; 344 345 if (vr->is_srcbuf) 346 msg = "srcbuf overwritten!"; 347 else if ((vr->pattern & PATTERN_COPY) 348 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 349 msg = "dstbuf not copied!"; 350 else if (diff & PATTERN_SRC) 351 msg = "dstbuf was copied!"; 352 else 353 msg = "dstbuf mismatch!"; 354 355 snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg, 356 vb->index, vb->expected, vb->actual); 357 358 return buf; 359 } 360 361 static char *thread_result_get(const char *name, 362 struct dmatest_thread_result *tr) 363 { 364 static const char * const messages[] = { 365 [DMATEST_ET_OK] = "No errors", 366 [DMATEST_ET_MAP_SRC] = "src mapping error", 367 [DMATEST_ET_MAP_DST] = "dst mapping error", 368 [DMATEST_ET_PREP] = "prep error", 369 [DMATEST_ET_SUBMIT] = "submit error", 370 [DMATEST_ET_TIMEOUT] = "test timed out", 371 [DMATEST_ET_DMA_ERROR] = 372 "got completion callback (DMA_ERROR)", 373 [DMATEST_ET_DMA_IN_PROGRESS] = 374 "got completion callback (DMA_IN_PROGRESS)", 375 [DMATEST_ET_VERIFY] = "errors", 376 [DMATEST_ET_VERIFY_BUF] = "verify errors", 377 }; 378 static char buf[512]; 379 380 snprintf(buf, sizeof(buf) - 1, 381 "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)", 382 name, tr->n, messages[tr->type], tr->src_off, tr->dst_off, 383 tr->len, tr->data); 384 385 return buf; 386 } 387 388 static int thread_result_add(struct dmatest_info *info, 389 struct dmatest_result *r, enum dmatest_error_type type, 390 unsigned int n, unsigned int src_off, unsigned int dst_off, 391 unsigned int len, unsigned long data) 392 { 393 struct dmatest_thread_result *tr; 394 395 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 396 if (!tr) 397 return -ENOMEM; 398 399 tr->type = type; 400 tr->n = n; 401 tr->src_off = src_off; 402 tr->dst_off = dst_off; 403 tr->len = len; 404 tr->data = data; 405 406 mutex_lock(&info->results_lock); 407 list_add_tail(&tr->node, &r->results); 408 mutex_unlock(&info->results_lock); 409 410 if (tr->type == DMATEST_ET_OK) 411 pr_debug("%s\n", thread_result_get(r->name, tr)); 412 else 413 pr_warn("%s\n", thread_result_get(r->name, tr)); 414 415 return 0; 416 } 417 418 static unsigned int verify_result_add(struct dmatest_info *info, 419 struct dmatest_result *r, unsigned int n, 420 unsigned int src_off, unsigned int dst_off, unsigned int len, 421 u8 **bufs, int whence, unsigned int counter, u8 pattern, 422 bool is_srcbuf) 423 { 424 struct dmatest_verify_result *vr; 425 unsigned int error_count; 426 unsigned int buf_off = is_srcbuf ? src_off : dst_off; 427 unsigned int start, end; 428 429 if (whence < 0) { 430 start = 0; 431 end = buf_off; 432 } else if (whence > 0) { 433 start = buf_off + len; 434 end = info->params.buf_size; 435 } else { 436 start = buf_off; 437 end = buf_off + len; 438 } 439 440 vr = kmalloc(sizeof(*vr), GFP_KERNEL); 441 if (!vr) { 442 pr_warn("dmatest: No memory to store verify result\n"); 443 return dmatest_verify(NULL, bufs, start, end, counter, pattern, 444 is_srcbuf); 445 } 446 447 vr->pattern = pattern; 448 vr->is_srcbuf = is_srcbuf; 449 450 error_count = dmatest_verify(vr, bufs, start, end, counter, pattern, 451 is_srcbuf); 452 if (error_count) { 453 vr->error_count = error_count; 454 thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off, 455 dst_off, len, (unsigned long)vr); 456 return error_count; 457 } 458 459 kfree(vr); 460 return 0; 461 } 462 463 static void result_free(struct dmatest_info *info, const char *name) 464 { 465 struct dmatest_result *r, *_r; 466 467 mutex_lock(&info->results_lock); 468 list_for_each_entry_safe(r, _r, &info->results, node) { 469 struct dmatest_thread_result *tr, *_tr; 470 471 if (name && strcmp(r->name, name)) 472 continue; 473 474 list_for_each_entry_safe(tr, _tr, &r->results, node) { 475 if (tr->type == DMATEST_ET_VERIFY_BUF) 476 kfree(tr->vr); 477 list_del(&tr->node); 478 kfree(tr); 479 } 480 481 kfree(r->name); 482 list_del(&r->node); 483 kfree(r); 484 } 485 486 mutex_unlock(&info->results_lock); 487 } 488 489 static struct dmatest_result *result_init(struct dmatest_info *info, 490 const char *name) 491 { 492 struct dmatest_result *r; 493 494 r = kzalloc(sizeof(*r), GFP_KERNEL); 495 if (r) { 496 r->name = kstrdup(name, GFP_KERNEL); 497 INIT_LIST_HEAD(&r->results); 498 mutex_lock(&info->results_lock); 499 list_add_tail(&r->node, &info->results); 500 mutex_unlock(&info->results_lock); 501 } 502 return r; 503 } 504 505 /* 506 * This function repeatedly tests DMA transfers of various lengths and 507 * offsets for a given operation type until it is told to exit by 508 * kthread_stop(). There may be multiple threads running this function 509 * in parallel for a single channel, and there may be multiple channels 510 * being tested in parallel. 511 * 512 * Before each test, the source and destination buffer is initialized 513 * with a known pattern. This pattern is different depending on 514 * whether it's in an area which is supposed to be copied or 515 * overwritten, and different in the source and destination buffers. 516 * So if the DMA engine doesn't copy exactly what we tell it to copy, 517 * we'll notice. 518 */ 519 static int dmatest_func(void *data) 520 { 521 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait); 522 struct dmatest_thread *thread = data; 523 struct dmatest_done done = { .wait = &done_wait }; 524 struct dmatest_info *info; 525 struct dmatest_params *params; 526 struct dma_chan *chan; 527 struct dma_device *dev; 528 const char *thread_name; 529 unsigned int src_off, dst_off, len; 530 unsigned int error_count; 531 unsigned int failed_tests = 0; 532 unsigned int total_tests = 0; 533 dma_cookie_t cookie; 534 enum dma_status status; 535 enum dma_ctrl_flags flags; 536 u8 *pq_coefs = NULL; 537 int ret; 538 int src_cnt; 539 int dst_cnt; 540 int i; 541 struct dmatest_result *result; 542 543 thread_name = current->comm; 544 set_freezable(); 545 546 ret = -ENOMEM; 547 548 smp_rmb(); 549 info = thread->info; 550 params = &info->params; 551 chan = thread->chan; 552 dev = chan->device; 553 if (thread->type == DMA_MEMCPY) 554 src_cnt = dst_cnt = 1; 555 else if (thread->type == DMA_XOR) { 556 /* force odd to ensure dst = src */ 557 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor); 558 dst_cnt = 1; 559 } else if (thread->type == DMA_PQ) { 560 /* force odd to ensure dst = src */ 561 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); 562 dst_cnt = 2; 563 564 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL); 565 if (!pq_coefs) 566 goto err_thread_type; 567 568 for (i = 0; i < src_cnt; i++) 569 pq_coefs[i] = 1; 570 } else 571 goto err_thread_type; 572 573 result = result_init(info, thread_name); 574 if (!result) 575 goto err_srcs; 576 577 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 578 if (!thread->srcs) 579 goto err_srcs; 580 for (i = 0; i < src_cnt; i++) { 581 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL); 582 if (!thread->srcs[i]) 583 goto err_srcbuf; 584 } 585 thread->srcs[i] = NULL; 586 587 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 588 if (!thread->dsts) 589 goto err_dsts; 590 for (i = 0; i < dst_cnt; i++) { 591 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL); 592 if (!thread->dsts[i]) 593 goto err_dstbuf; 594 } 595 thread->dsts[i] = NULL; 596 597 set_user_nice(current, 10); 598 599 /* 600 * src buffers are freed by the DMAEngine code with dma_unmap_single() 601 * dst buffers are freed by ourselves below 602 */ 603 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT 604 | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE; 605 606 while (!kthread_should_stop() 607 && !(params->iterations && total_tests >= params->iterations)) { 608 struct dma_async_tx_descriptor *tx = NULL; 609 dma_addr_t dma_srcs[src_cnt]; 610 dma_addr_t dma_dsts[dst_cnt]; 611 u8 align = 0; 612 613 total_tests++; 614 615 /* honor alignment restrictions */ 616 if (thread->type == DMA_MEMCPY) 617 align = dev->copy_align; 618 else if (thread->type == DMA_XOR) 619 align = dev->xor_align; 620 else if (thread->type == DMA_PQ) 621 align = dev->pq_align; 622 623 if (1 << align > params->buf_size) { 624 pr_err("%u-byte buffer too small for %d-byte alignment\n", 625 params->buf_size, 1 << align); 626 break; 627 } 628 629 len = dmatest_random() % params->buf_size + 1; 630 len = (len >> align) << align; 631 if (!len) 632 len = 1 << align; 633 src_off = dmatest_random() % (params->buf_size - len + 1); 634 dst_off = dmatest_random() % (params->buf_size - len + 1); 635 636 src_off = (src_off >> align) << align; 637 dst_off = (dst_off >> align) << align; 638 639 dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size); 640 dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size); 641 642 for (i = 0; i < src_cnt; i++) { 643 u8 *buf = thread->srcs[i] + src_off; 644 645 dma_srcs[i] = dma_map_single(dev->dev, buf, len, 646 DMA_TO_DEVICE); 647 ret = dma_mapping_error(dev->dev, dma_srcs[i]); 648 if (ret) { 649 unmap_src(dev->dev, dma_srcs, len, i); 650 thread_result_add(info, result, 651 DMATEST_ET_MAP_SRC, 652 total_tests, src_off, dst_off, 653 len, ret); 654 failed_tests++; 655 continue; 656 } 657 } 658 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 659 for (i = 0; i < dst_cnt; i++) { 660 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 661 params->buf_size, 662 DMA_BIDIRECTIONAL); 663 ret = dma_mapping_error(dev->dev, dma_dsts[i]); 664 if (ret) { 665 unmap_src(dev->dev, dma_srcs, len, src_cnt); 666 unmap_dst(dev->dev, dma_dsts, params->buf_size, 667 i); 668 thread_result_add(info, result, 669 DMATEST_ET_MAP_DST, 670 total_tests, src_off, dst_off, 671 len, ret); 672 failed_tests++; 673 continue; 674 } 675 } 676 677 if (thread->type == DMA_MEMCPY) 678 tx = dev->device_prep_dma_memcpy(chan, 679 dma_dsts[0] + dst_off, 680 dma_srcs[0], len, 681 flags); 682 else if (thread->type == DMA_XOR) 683 tx = dev->device_prep_dma_xor(chan, 684 dma_dsts[0] + dst_off, 685 dma_srcs, src_cnt, 686 len, flags); 687 else if (thread->type == DMA_PQ) { 688 dma_addr_t dma_pq[dst_cnt]; 689 690 for (i = 0; i < dst_cnt; i++) 691 dma_pq[i] = dma_dsts[i] + dst_off; 692 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, 693 src_cnt, pq_coefs, 694 len, flags); 695 } 696 697 if (!tx) { 698 unmap_src(dev->dev, dma_srcs, len, src_cnt); 699 unmap_dst(dev->dev, dma_dsts, params->buf_size, 700 dst_cnt); 701 thread_result_add(info, result, DMATEST_ET_PREP, 702 total_tests, src_off, dst_off, 703 len, 0); 704 msleep(100); 705 failed_tests++; 706 continue; 707 } 708 709 done.done = false; 710 tx->callback = dmatest_callback; 711 tx->callback_param = &done; 712 cookie = tx->tx_submit(tx); 713 714 if (dma_submit_error(cookie)) { 715 thread_result_add(info, result, DMATEST_ET_SUBMIT, 716 total_tests, src_off, dst_off, 717 len, cookie); 718 msleep(100); 719 failed_tests++; 720 continue; 721 } 722 dma_async_issue_pending(chan); 723 724 wait_event_freezable_timeout(done_wait, done.done, 725 msecs_to_jiffies(params->timeout)); 726 727 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 728 729 if (!done.done) { 730 /* 731 * We're leaving the timed out dma operation with 732 * dangling pointer to done_wait. To make this 733 * correct, we'll need to allocate wait_done for 734 * each test iteration and perform "who's gonna 735 * free it this time?" dancing. For now, just 736 * leave it dangling. 737 */ 738 thread_result_add(info, result, DMATEST_ET_TIMEOUT, 739 total_tests, src_off, dst_off, 740 len, 0); 741 failed_tests++; 742 continue; 743 } else if (status != DMA_SUCCESS) { 744 enum dmatest_error_type type = (status == DMA_ERROR) ? 745 DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS; 746 thread_result_add(info, result, type, 747 total_tests, src_off, dst_off, 748 len, status); 749 failed_tests++; 750 continue; 751 } 752 753 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 754 unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt); 755 756 error_count = 0; 757 758 pr_debug("%s: verifying source buffer...\n", thread_name); 759 error_count += verify_result_add(info, result, total_tests, 760 src_off, dst_off, len, thread->srcs, -1, 761 0, PATTERN_SRC, true); 762 error_count += verify_result_add(info, result, total_tests, 763 src_off, dst_off, len, thread->srcs, 0, 764 src_off, PATTERN_SRC | PATTERN_COPY, true); 765 error_count += verify_result_add(info, result, total_tests, 766 src_off, dst_off, len, thread->srcs, 1, 767 src_off + len, PATTERN_SRC, true); 768 769 pr_debug("%s: verifying dest buffer...\n", thread_name); 770 error_count += verify_result_add(info, result, total_tests, 771 src_off, dst_off, len, thread->dsts, -1, 772 0, PATTERN_DST, false); 773 error_count += verify_result_add(info, result, total_tests, 774 src_off, dst_off, len, thread->dsts, 0, 775 src_off, PATTERN_SRC | PATTERN_COPY, false); 776 error_count += verify_result_add(info, result, total_tests, 777 src_off, dst_off, len, thread->dsts, 1, 778 dst_off + len, PATTERN_DST, false); 779 780 if (error_count) { 781 thread_result_add(info, result, DMATEST_ET_VERIFY, 782 total_tests, src_off, dst_off, 783 len, error_count); 784 failed_tests++; 785 } else { 786 thread_result_add(info, result, DMATEST_ET_OK, 787 total_tests, src_off, dst_off, 788 len, 0); 789 } 790 } 791 792 ret = 0; 793 for (i = 0; thread->dsts[i]; i++) 794 kfree(thread->dsts[i]); 795 err_dstbuf: 796 kfree(thread->dsts); 797 err_dsts: 798 for (i = 0; thread->srcs[i]; i++) 799 kfree(thread->srcs[i]); 800 err_srcbuf: 801 kfree(thread->srcs); 802 err_srcs: 803 kfree(pq_coefs); 804 err_thread_type: 805 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 806 thread_name, total_tests, failed_tests, ret); 807 808 /* terminate all transfers on specified channels */ 809 if (ret) 810 dmaengine_terminate_all(chan); 811 812 thread->done = true; 813 814 if (params->iterations > 0) 815 while (!kthread_should_stop()) { 816 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); 817 interruptible_sleep_on(&wait_dmatest_exit); 818 } 819 820 return ret; 821 } 822 823 static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 824 { 825 struct dmatest_thread *thread; 826 struct dmatest_thread *_thread; 827 int ret; 828 829 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 830 ret = kthread_stop(thread->task); 831 pr_debug("dmatest: thread %s exited with status %d\n", 832 thread->task->comm, ret); 833 list_del(&thread->node); 834 kfree(thread); 835 } 836 837 /* terminate all transfers on specified channels */ 838 dmaengine_terminate_all(dtc->chan); 839 840 kfree(dtc); 841 } 842 843 static int dmatest_add_threads(struct dmatest_info *info, 844 struct dmatest_chan *dtc, enum dma_transaction_type type) 845 { 846 struct dmatest_params *params = &info->params; 847 struct dmatest_thread *thread; 848 struct dma_chan *chan = dtc->chan; 849 char *op; 850 unsigned int i; 851 852 if (type == DMA_MEMCPY) 853 op = "copy"; 854 else if (type == DMA_XOR) 855 op = "xor"; 856 else if (type == DMA_PQ) 857 op = "pq"; 858 else 859 return -EINVAL; 860 861 for (i = 0; i < params->threads_per_chan; i++) { 862 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 863 if (!thread) { 864 pr_warning("dmatest: No memory for %s-%s%u\n", 865 dma_chan_name(chan), op, i); 866 867 break; 868 } 869 thread->info = info; 870 thread->chan = dtc->chan; 871 thread->type = type; 872 smp_wmb(); 873 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 874 dma_chan_name(chan), op, i); 875 if (IS_ERR(thread->task)) { 876 pr_warning("dmatest: Failed to run thread %s-%s%u\n", 877 dma_chan_name(chan), op, i); 878 kfree(thread); 879 break; 880 } 881 882 /* srcbuf and dstbuf are allocated by the thread itself */ 883 884 list_add_tail(&thread->node, &dtc->threads); 885 } 886 887 return i; 888 } 889 890 static int dmatest_add_channel(struct dmatest_info *info, 891 struct dma_chan *chan) 892 { 893 struct dmatest_chan *dtc; 894 struct dma_device *dma_dev = chan->device; 895 unsigned int thread_count = 0; 896 int cnt; 897 898 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 899 if (!dtc) { 900 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 901 return -ENOMEM; 902 } 903 904 dtc->chan = chan; 905 INIT_LIST_HEAD(&dtc->threads); 906 907 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 908 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY); 909 thread_count += cnt > 0 ? cnt : 0; 910 } 911 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 912 cnt = dmatest_add_threads(info, dtc, DMA_XOR); 913 thread_count += cnt > 0 ? cnt : 0; 914 } 915 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 916 cnt = dmatest_add_threads(info, dtc, DMA_PQ); 917 thread_count += cnt > 0 ? cnt : 0; 918 } 919 920 pr_info("dmatest: Started %u threads using %s\n", 921 thread_count, dma_chan_name(chan)); 922 923 list_add_tail(&dtc->node, &info->channels); 924 info->nr_channels++; 925 926 return 0; 927 } 928 929 static bool filter(struct dma_chan *chan, void *param) 930 { 931 struct dmatest_params *params = param; 932 933 if (!dmatest_match_channel(params, chan) || 934 !dmatest_match_device(params, chan->device)) 935 return false; 936 else 937 return true; 938 } 939 940 static int __run_threaded_test(struct dmatest_info *info) 941 { 942 dma_cap_mask_t mask; 943 struct dma_chan *chan; 944 struct dmatest_params *params = &info->params; 945 int err = 0; 946 947 dma_cap_zero(mask); 948 dma_cap_set(DMA_MEMCPY, mask); 949 for (;;) { 950 chan = dma_request_channel(mask, filter, params); 951 if (chan) { 952 err = dmatest_add_channel(info, chan); 953 if (err) { 954 dma_release_channel(chan); 955 break; /* add_channel failed, punt */ 956 } 957 } else 958 break; /* no more channels available */ 959 if (params->max_channels && 960 info->nr_channels >= params->max_channels) 961 break; /* we have all we need */ 962 } 963 return err; 964 } 965 966 #ifndef MODULE 967 static int run_threaded_test(struct dmatest_info *info) 968 { 969 int ret; 970 971 mutex_lock(&info->lock); 972 ret = __run_threaded_test(info); 973 mutex_unlock(&info->lock); 974 return ret; 975 } 976 #endif 977 978 static void __stop_threaded_test(struct dmatest_info *info) 979 { 980 struct dmatest_chan *dtc, *_dtc; 981 struct dma_chan *chan; 982 983 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { 984 list_del(&dtc->node); 985 chan = dtc->chan; 986 dmatest_cleanup_channel(dtc); 987 pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan)); 988 dma_release_channel(chan); 989 } 990 991 info->nr_channels = 0; 992 } 993 994 static void stop_threaded_test(struct dmatest_info *info) 995 { 996 mutex_lock(&info->lock); 997 __stop_threaded_test(info); 998 mutex_unlock(&info->lock); 999 } 1000 1001 static int __restart_threaded_test(struct dmatest_info *info, bool run) 1002 { 1003 struct dmatest_params *params = &info->params; 1004 1005 /* Stop any running test first */ 1006 __stop_threaded_test(info); 1007 1008 if (run == false) 1009 return 0; 1010 1011 /* Clear results from previous run */ 1012 result_free(info, NULL); 1013 1014 /* Copy test parameters */ 1015 params->buf_size = test_buf_size; 1016 strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); 1017 strlcpy(params->device, strim(test_device), sizeof(params->device)); 1018 params->threads_per_chan = threads_per_chan; 1019 params->max_channels = max_channels; 1020 params->iterations = iterations; 1021 params->xor_sources = xor_sources; 1022 params->pq_sources = pq_sources; 1023 params->timeout = timeout; 1024 1025 /* Run test with new parameters */ 1026 return __run_threaded_test(info); 1027 } 1028 1029 static bool __is_threaded_test_run(struct dmatest_info *info) 1030 { 1031 struct dmatest_chan *dtc; 1032 1033 list_for_each_entry(dtc, &info->channels, node) { 1034 struct dmatest_thread *thread; 1035 1036 list_for_each_entry(thread, &dtc->threads, node) { 1037 if (!thread->done) 1038 return true; 1039 } 1040 } 1041 1042 return false; 1043 } 1044 1045 static ssize_t dtf_read_run(struct file *file, char __user *user_buf, 1046 size_t count, loff_t *ppos) 1047 { 1048 struct dmatest_info *info = file->private_data; 1049 char buf[3]; 1050 1051 mutex_lock(&info->lock); 1052 1053 if (__is_threaded_test_run(info)) { 1054 buf[0] = 'Y'; 1055 } else { 1056 __stop_threaded_test(info); 1057 buf[0] = 'N'; 1058 } 1059 1060 mutex_unlock(&info->lock); 1061 buf[1] = '\n'; 1062 buf[2] = 0x00; 1063 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 1064 } 1065 1066 static ssize_t dtf_write_run(struct file *file, const char __user *user_buf, 1067 size_t count, loff_t *ppos) 1068 { 1069 struct dmatest_info *info = file->private_data; 1070 char buf[16]; 1071 bool bv; 1072 int ret = 0; 1073 1074 if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1)))) 1075 return -EFAULT; 1076 1077 if (strtobool(buf, &bv) == 0) { 1078 mutex_lock(&info->lock); 1079 1080 if (__is_threaded_test_run(info)) 1081 ret = -EBUSY; 1082 else 1083 ret = __restart_threaded_test(info, bv); 1084 1085 mutex_unlock(&info->lock); 1086 } 1087 1088 return ret ? ret : count; 1089 } 1090 1091 static const struct file_operations dtf_run_fops = { 1092 .read = dtf_read_run, 1093 .write = dtf_write_run, 1094 .open = simple_open, 1095 .llseek = default_llseek, 1096 }; 1097 1098 static int dtf_results_show(struct seq_file *sf, void *data) 1099 { 1100 struct dmatest_info *info = sf->private; 1101 struct dmatest_result *result; 1102 struct dmatest_thread_result *tr; 1103 unsigned int i; 1104 1105 mutex_lock(&info->results_lock); 1106 list_for_each_entry(result, &info->results, node) { 1107 list_for_each_entry(tr, &result->results, node) { 1108 seq_printf(sf, "%s\n", 1109 thread_result_get(result->name, tr)); 1110 if (tr->type == DMATEST_ET_VERIFY_BUF) { 1111 for (i = 0; i < tr->vr->error_count; i++) { 1112 seq_printf(sf, "\t%s\n", 1113 verify_result_get_one(tr->vr, i)); 1114 } 1115 } 1116 } 1117 } 1118 1119 mutex_unlock(&info->results_lock); 1120 return 0; 1121 } 1122 1123 static int dtf_results_open(struct inode *inode, struct file *file) 1124 { 1125 return single_open(file, dtf_results_show, inode->i_private); 1126 } 1127 1128 static const struct file_operations dtf_results_fops = { 1129 .open = dtf_results_open, 1130 .read = seq_read, 1131 .llseek = seq_lseek, 1132 .release = single_release, 1133 }; 1134 1135 static int dmatest_register_dbgfs(struct dmatest_info *info) 1136 { 1137 struct dentry *d; 1138 1139 d = debugfs_create_dir("dmatest", NULL); 1140 if (IS_ERR(d)) 1141 return PTR_ERR(d); 1142 if (!d) 1143 goto err_root; 1144 1145 info->root = d; 1146 1147 /* Run or stop threaded test */ 1148 debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root, info, 1149 &dtf_run_fops); 1150 1151 /* Results of test in progress */ 1152 debugfs_create_file("results", S_IRUGO, info->root, info, 1153 &dtf_results_fops); 1154 1155 return 0; 1156 1157 err_root: 1158 pr_err("dmatest: Failed to initialize debugfs\n"); 1159 return -ENOMEM; 1160 } 1161 1162 static int __init dmatest_init(void) 1163 { 1164 struct dmatest_info *info = &test_info; 1165 int ret; 1166 1167 memset(info, 0, sizeof(*info)); 1168 1169 mutex_init(&info->lock); 1170 INIT_LIST_HEAD(&info->channels); 1171 1172 mutex_init(&info->results_lock); 1173 INIT_LIST_HEAD(&info->results); 1174 1175 ret = dmatest_register_dbgfs(info); 1176 if (ret) 1177 return ret; 1178 1179 #ifdef MODULE 1180 return 0; 1181 #else 1182 return run_threaded_test(info); 1183 #endif 1184 } 1185 /* when compiled-in wait for drivers to load first */ 1186 late_initcall(dmatest_init); 1187 1188 static void __exit dmatest_exit(void) 1189 { 1190 struct dmatest_info *info = &test_info; 1191 1192 debugfs_remove_recursive(info->root); 1193 stop_threaded_test(info); 1194 result_free(info, NULL); 1195 } 1196 module_exit(dmatest_exit); 1197 1198 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 1199 MODULE_LICENSE("GPL v2"); 1200