1 /* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * Copyright (C) 2013 Intel Corporation 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/delay.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/dmaengine.h> 16 #include <linux/freezer.h> 17 #include <linux/init.h> 18 #include <linux/kthread.h> 19 #include <linux/sched/task.h> 20 #include <linux/module.h> 21 #include <linux/moduleparam.h> 22 #include <linux/random.h> 23 #include <linux/slab.h> 24 #include <linux/wait.h> 25 26 static unsigned int test_buf_size = 16384; 27 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR); 28 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 29 30 static char test_channel[20]; 31 module_param_string(channel, test_channel, sizeof(test_channel), 32 S_IRUGO | S_IWUSR); 33 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 34 35 static char test_device[32]; 36 module_param_string(device, test_device, sizeof(test_device), 37 S_IRUGO | S_IWUSR); 38 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 39 40 static unsigned int threads_per_chan = 1; 41 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR); 42 MODULE_PARM_DESC(threads_per_chan, 43 "Number of threads to start per channel (default: 1)"); 44 45 static unsigned int max_channels; 46 module_param(max_channels, uint, S_IRUGO | S_IWUSR); 47 MODULE_PARM_DESC(max_channels, 48 "Maximum number of channels to use (default: all)"); 49 50 static unsigned int iterations; 51 module_param(iterations, uint, S_IRUGO | S_IWUSR); 52 MODULE_PARM_DESC(iterations, 53 "Iterations before stopping test (default: infinite)"); 54 55 static unsigned int dmatest; 56 module_param(dmatest, uint, S_IRUGO | S_IWUSR); 57 MODULE_PARM_DESC(dmatest, 58 "dmatest 0-memcpy 1-memset (default: 0)"); 59 60 static unsigned int xor_sources = 3; 61 module_param(xor_sources, uint, S_IRUGO | S_IWUSR); 62 MODULE_PARM_DESC(xor_sources, 63 "Number of xor source buffers (default: 3)"); 64 65 static unsigned int pq_sources = 3; 66 module_param(pq_sources, uint, S_IRUGO | S_IWUSR); 67 MODULE_PARM_DESC(pq_sources, 68 "Number of p+q source buffers (default: 3)"); 69 70 static int timeout = 3000; 71 module_param(timeout, uint, S_IRUGO | S_IWUSR); 72 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 73 "Pass -1 for infinite timeout"); 74 75 static bool noverify; 76 module_param(noverify, bool, S_IRUGO | S_IWUSR); 77 MODULE_PARM_DESC(noverify, "Disable random data setup and verification"); 78 79 static bool verbose; 80 module_param(verbose, bool, S_IRUGO | S_IWUSR); 81 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)"); 82 83 /** 84 * struct dmatest_params - test parameters. 85 * @buf_size: size of the memcpy test buffer 86 * @channel: bus ID of the channel to test 87 * @device: bus ID of the DMA Engine to test 88 * @threads_per_chan: number of threads to start per channel 89 * @max_channels: maximum number of channels to use 90 * @iterations: iterations before stopping test 91 * @xor_sources: number of xor source buffers 92 * @pq_sources: number of p+q source buffers 93 * @timeout: transfer timeout in msec, -1 for infinite timeout 94 */ 95 struct dmatest_params { 96 unsigned int buf_size; 97 char channel[20]; 98 char device[32]; 99 unsigned int threads_per_chan; 100 unsigned int max_channels; 101 unsigned int iterations; 102 unsigned int xor_sources; 103 unsigned int pq_sources; 104 int timeout; 105 bool noverify; 106 }; 107 108 /** 109 * struct dmatest_info - test information. 110 * @params: test parameters 111 * @lock: access protection to the fields of this structure 112 */ 113 static struct dmatest_info { 114 /* Test parameters */ 115 struct dmatest_params params; 116 117 /* Internal state */ 118 struct list_head channels; 119 unsigned int nr_channels; 120 struct mutex lock; 121 bool did_init; 122 } test_info = { 123 .channels = LIST_HEAD_INIT(test_info.channels), 124 .lock = __MUTEX_INITIALIZER(test_info.lock), 125 }; 126 127 static int dmatest_run_set(const char *val, const struct kernel_param *kp); 128 static int dmatest_run_get(char *val, const struct kernel_param *kp); 129 static const struct kernel_param_ops run_ops = { 130 .set = dmatest_run_set, 131 .get = dmatest_run_get, 132 }; 133 static bool dmatest_run; 134 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR); 135 MODULE_PARM_DESC(run, "Run the test (default: false)"); 136 137 /* Maximum amount of mismatched bytes in buffer to print */ 138 #define MAX_ERROR_COUNT 32 139 140 /* 141 * Initialization patterns. All bytes in the source buffer has bit 7 142 * set, all bytes in the destination buffer has bit 7 cleared. 143 * 144 * Bit 6 is set for all bytes which are to be copied by the DMA 145 * engine. Bit 5 is set for all bytes which are to be overwritten by 146 * the DMA engine. 147 * 148 * The remaining bits are the inverse of a counter which increments by 149 * one for each byte address. 150 */ 151 #define PATTERN_SRC 0x80 152 #define PATTERN_DST 0x00 153 #define PATTERN_COPY 0x40 154 #define PATTERN_OVERWRITE 0x20 155 #define PATTERN_COUNT_MASK 0x1f 156 #define PATTERN_MEMSET_IDX 0x01 157 158 struct dmatest_thread { 159 struct list_head node; 160 struct dmatest_info *info; 161 struct task_struct *task; 162 struct dma_chan *chan; 163 u8 **srcs; 164 u8 **usrcs; 165 u8 **dsts; 166 u8 **udsts; 167 enum dma_transaction_type type; 168 bool done; 169 }; 170 171 struct dmatest_chan { 172 struct list_head node; 173 struct dma_chan *chan; 174 struct list_head threads; 175 }; 176 177 static DECLARE_WAIT_QUEUE_HEAD(thread_wait); 178 static bool wait; 179 180 static bool is_threaded_test_run(struct dmatest_info *info) 181 { 182 struct dmatest_chan *dtc; 183 184 list_for_each_entry(dtc, &info->channels, node) { 185 struct dmatest_thread *thread; 186 187 list_for_each_entry(thread, &dtc->threads, node) { 188 if (!thread->done) 189 return true; 190 } 191 } 192 193 return false; 194 } 195 196 static int dmatest_wait_get(char *val, const struct kernel_param *kp) 197 { 198 struct dmatest_info *info = &test_info; 199 struct dmatest_params *params = &info->params; 200 201 if (params->iterations) 202 wait_event(thread_wait, !is_threaded_test_run(info)); 203 wait = true; 204 return param_get_bool(val, kp); 205 } 206 207 static const struct kernel_param_ops wait_ops = { 208 .get = dmatest_wait_get, 209 .set = param_set_bool, 210 }; 211 module_param_cb(wait, &wait_ops, &wait, S_IRUGO); 212 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)"); 213 214 static bool dmatest_match_channel(struct dmatest_params *params, 215 struct dma_chan *chan) 216 { 217 if (params->channel[0] == '\0') 218 return true; 219 return strcmp(dma_chan_name(chan), params->channel) == 0; 220 } 221 222 static bool dmatest_match_device(struct dmatest_params *params, 223 struct dma_device *device) 224 { 225 if (params->device[0] == '\0') 226 return true; 227 return strcmp(dev_name(device->dev), params->device) == 0; 228 } 229 230 static unsigned long dmatest_random(void) 231 { 232 unsigned long buf; 233 234 prandom_bytes(&buf, sizeof(buf)); 235 return buf; 236 } 237 238 static inline u8 gen_inv_idx(u8 index, bool is_memset) 239 { 240 u8 val = is_memset ? PATTERN_MEMSET_IDX : index; 241 242 return ~val & PATTERN_COUNT_MASK; 243 } 244 245 static inline u8 gen_src_value(u8 index, bool is_memset) 246 { 247 return PATTERN_SRC | gen_inv_idx(index, is_memset); 248 } 249 250 static inline u8 gen_dst_value(u8 index, bool is_memset) 251 { 252 return PATTERN_DST | gen_inv_idx(index, is_memset); 253 } 254 255 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len, 256 unsigned int buf_size, bool is_memset) 257 { 258 unsigned int i; 259 u8 *buf; 260 261 for (; (buf = *bufs); bufs++) { 262 for (i = 0; i < start; i++) 263 buf[i] = gen_src_value(i, is_memset); 264 for ( ; i < start + len; i++) 265 buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY; 266 for ( ; i < buf_size; i++) 267 buf[i] = gen_src_value(i, is_memset); 268 buf++; 269 } 270 } 271 272 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len, 273 unsigned int buf_size, bool is_memset) 274 { 275 unsigned int i; 276 u8 *buf; 277 278 for (; (buf = *bufs); bufs++) { 279 for (i = 0; i < start; i++) 280 buf[i] = gen_dst_value(i, is_memset); 281 for ( ; i < start + len; i++) 282 buf[i] = gen_dst_value(i, is_memset) | 283 PATTERN_OVERWRITE; 284 for ( ; i < buf_size; i++) 285 buf[i] = gen_dst_value(i, is_memset); 286 } 287 } 288 289 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 290 unsigned int counter, bool is_srcbuf, bool is_memset) 291 { 292 u8 diff = actual ^ pattern; 293 u8 expected = pattern | gen_inv_idx(counter, is_memset); 294 const char *thread_name = current->comm; 295 296 if (is_srcbuf) 297 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n", 298 thread_name, index, expected, actual); 299 else if ((pattern & PATTERN_COPY) 300 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 301 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n", 302 thread_name, index, expected, actual); 303 else if (diff & PATTERN_SRC) 304 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n", 305 thread_name, index, expected, actual); 306 else 307 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n", 308 thread_name, index, expected, actual); 309 } 310 311 static unsigned int dmatest_verify(u8 **bufs, unsigned int start, 312 unsigned int end, unsigned int counter, u8 pattern, 313 bool is_srcbuf, bool is_memset) 314 { 315 unsigned int i; 316 unsigned int error_count = 0; 317 u8 actual; 318 u8 expected; 319 u8 *buf; 320 unsigned int counter_orig = counter; 321 322 for (; (buf = *bufs); bufs++) { 323 counter = counter_orig; 324 for (i = start; i < end; i++) { 325 actual = buf[i]; 326 expected = pattern | gen_inv_idx(counter, is_memset); 327 if (actual != expected) { 328 if (error_count < MAX_ERROR_COUNT) 329 dmatest_mismatch(actual, pattern, i, 330 counter, is_srcbuf, 331 is_memset); 332 error_count++; 333 } 334 counter++; 335 } 336 } 337 338 if (error_count > MAX_ERROR_COUNT) 339 pr_warn("%s: %u errors suppressed\n", 340 current->comm, error_count - MAX_ERROR_COUNT); 341 342 return error_count; 343 } 344 345 /* poor man's completion - we want to use wait_event_freezable() on it */ 346 struct dmatest_done { 347 bool done; 348 wait_queue_head_t *wait; 349 }; 350 351 static void dmatest_callback(void *arg) 352 { 353 struct dmatest_done *done = arg; 354 355 done->done = true; 356 wake_up_all(done->wait); 357 } 358 359 static unsigned int min_odd(unsigned int x, unsigned int y) 360 { 361 unsigned int val = min(x, y); 362 363 return val % 2 ? val : val - 1; 364 } 365 366 static void result(const char *err, unsigned int n, unsigned int src_off, 367 unsigned int dst_off, unsigned int len, unsigned long data) 368 { 369 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n", 370 current->comm, n, err, src_off, dst_off, len, data); 371 } 372 373 static void dbg_result(const char *err, unsigned int n, unsigned int src_off, 374 unsigned int dst_off, unsigned int len, 375 unsigned long data) 376 { 377 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n", 378 current->comm, n, err, src_off, dst_off, len, data); 379 } 380 381 #define verbose_result(err, n, src_off, dst_off, len, data) ({ \ 382 if (verbose) \ 383 result(err, n, src_off, dst_off, len, data); \ 384 else \ 385 dbg_result(err, n, src_off, dst_off, len, data);\ 386 }) 387 388 static unsigned long long dmatest_persec(s64 runtime, unsigned int val) 389 { 390 unsigned long long per_sec = 1000000; 391 392 if (runtime <= 0) 393 return 0; 394 395 /* drop precision until runtime is 32-bits */ 396 while (runtime > UINT_MAX) { 397 runtime >>= 1; 398 per_sec <<= 1; 399 } 400 401 per_sec *= val; 402 do_div(per_sec, runtime); 403 return per_sec; 404 } 405 406 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len) 407 { 408 return dmatest_persec(runtime, len >> 10); 409 } 410 411 /* 412 * This function repeatedly tests DMA transfers of various lengths and 413 * offsets for a given operation type until it is told to exit by 414 * kthread_stop(). There may be multiple threads running this function 415 * in parallel for a single channel, and there may be multiple channels 416 * being tested in parallel. 417 * 418 * Before each test, the source and destination buffer is initialized 419 * with a known pattern. This pattern is different depending on 420 * whether it's in an area which is supposed to be copied or 421 * overwritten, and different in the source and destination buffers. 422 * So if the DMA engine doesn't copy exactly what we tell it to copy, 423 * we'll notice. 424 */ 425 static int dmatest_func(void *data) 426 { 427 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait); 428 struct dmatest_thread *thread = data; 429 struct dmatest_done done = { .wait = &done_wait }; 430 struct dmatest_info *info; 431 struct dmatest_params *params; 432 struct dma_chan *chan; 433 struct dma_device *dev; 434 unsigned int error_count; 435 unsigned int failed_tests = 0; 436 unsigned int total_tests = 0; 437 dma_cookie_t cookie; 438 enum dma_status status; 439 enum dma_ctrl_flags flags; 440 u8 *pq_coefs = NULL; 441 int ret; 442 int src_cnt; 443 int dst_cnt; 444 int i; 445 ktime_t ktime, start, diff; 446 ktime_t filltime = 0; 447 ktime_t comparetime = 0; 448 s64 runtime = 0; 449 unsigned long long total_len = 0; 450 u8 align = 0; 451 bool is_memset = false; 452 453 set_freezable(); 454 455 ret = -ENOMEM; 456 457 smp_rmb(); 458 info = thread->info; 459 params = &info->params; 460 chan = thread->chan; 461 dev = chan->device; 462 if (thread->type == DMA_MEMCPY) { 463 align = dev->copy_align; 464 src_cnt = dst_cnt = 1; 465 } else if (thread->type == DMA_MEMSET) { 466 align = dev->fill_align; 467 src_cnt = dst_cnt = 1; 468 is_memset = true; 469 } else if (thread->type == DMA_XOR) { 470 /* force odd to ensure dst = src */ 471 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor); 472 dst_cnt = 1; 473 align = dev->xor_align; 474 } else if (thread->type == DMA_PQ) { 475 /* force odd to ensure dst = src */ 476 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); 477 dst_cnt = 2; 478 align = dev->pq_align; 479 480 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL); 481 if (!pq_coefs) 482 goto err_thread_type; 483 484 for (i = 0; i < src_cnt; i++) 485 pq_coefs[i] = 1; 486 } else 487 goto err_thread_type; 488 489 thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL); 490 if (!thread->srcs) 491 goto err_srcs; 492 493 thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL); 494 if (!thread->usrcs) 495 goto err_usrcs; 496 497 for (i = 0; i < src_cnt; i++) { 498 thread->usrcs[i] = kmalloc(params->buf_size + align, 499 GFP_KERNEL); 500 if (!thread->usrcs[i]) 501 goto err_srcbuf; 502 503 /* align srcs to alignment restriction */ 504 if (align) 505 thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align); 506 else 507 thread->srcs[i] = thread->usrcs[i]; 508 } 509 thread->srcs[i] = NULL; 510 511 thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL); 512 if (!thread->dsts) 513 goto err_dsts; 514 515 thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL); 516 if (!thread->udsts) 517 goto err_udsts; 518 519 for (i = 0; i < dst_cnt; i++) { 520 thread->udsts[i] = kmalloc(params->buf_size + align, 521 GFP_KERNEL); 522 if (!thread->udsts[i]) 523 goto err_dstbuf; 524 525 /* align dsts to alignment restriction */ 526 if (align) 527 thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align); 528 else 529 thread->dsts[i] = thread->udsts[i]; 530 } 531 thread->dsts[i] = NULL; 532 533 set_user_nice(current, 10); 534 535 /* 536 * src and dst buffers are freed by ourselves below 537 */ 538 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; 539 540 ktime = ktime_get(); 541 while (!kthread_should_stop() 542 && !(params->iterations && total_tests >= params->iterations)) { 543 struct dma_async_tx_descriptor *tx = NULL; 544 struct dmaengine_unmap_data *um; 545 dma_addr_t srcs[src_cnt]; 546 dma_addr_t *dsts; 547 unsigned int src_off, dst_off, len; 548 549 total_tests++; 550 551 /* Check if buffer count fits into map count variable (u8) */ 552 if ((src_cnt + dst_cnt) >= 255) { 553 pr_err("too many buffers (%d of 255 supported)\n", 554 src_cnt + dst_cnt); 555 break; 556 } 557 558 if (1 << align > params->buf_size) { 559 pr_err("%u-byte buffer too small for %d-byte alignment\n", 560 params->buf_size, 1 << align); 561 break; 562 } 563 564 if (params->noverify) 565 len = params->buf_size; 566 else 567 len = dmatest_random() % params->buf_size + 1; 568 569 len = (len >> align) << align; 570 if (!len) 571 len = 1 << align; 572 573 total_len += len; 574 575 if (params->noverify) { 576 src_off = 0; 577 dst_off = 0; 578 } else { 579 start = ktime_get(); 580 src_off = dmatest_random() % (params->buf_size - len + 1); 581 dst_off = dmatest_random() % (params->buf_size - len + 1); 582 583 src_off = (src_off >> align) << align; 584 dst_off = (dst_off >> align) << align; 585 586 dmatest_init_srcs(thread->srcs, src_off, len, 587 params->buf_size, is_memset); 588 dmatest_init_dsts(thread->dsts, dst_off, len, 589 params->buf_size, is_memset); 590 591 diff = ktime_sub(ktime_get(), start); 592 filltime = ktime_add(filltime, diff); 593 } 594 595 um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt, 596 GFP_KERNEL); 597 if (!um) { 598 failed_tests++; 599 result("unmap data NULL", total_tests, 600 src_off, dst_off, len, ret); 601 continue; 602 } 603 604 um->len = params->buf_size; 605 for (i = 0; i < src_cnt; i++) { 606 void *buf = thread->srcs[i]; 607 struct page *pg = virt_to_page(buf); 608 unsigned long pg_off = offset_in_page(buf); 609 610 um->addr[i] = dma_map_page(dev->dev, pg, pg_off, 611 um->len, DMA_TO_DEVICE); 612 srcs[i] = um->addr[i] + src_off; 613 ret = dma_mapping_error(dev->dev, um->addr[i]); 614 if (ret) { 615 dmaengine_unmap_put(um); 616 result("src mapping error", total_tests, 617 src_off, dst_off, len, ret); 618 failed_tests++; 619 continue; 620 } 621 um->to_cnt++; 622 } 623 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 624 dsts = &um->addr[src_cnt]; 625 for (i = 0; i < dst_cnt; i++) { 626 void *buf = thread->dsts[i]; 627 struct page *pg = virt_to_page(buf); 628 unsigned long pg_off = offset_in_page(buf); 629 630 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len, 631 DMA_BIDIRECTIONAL); 632 ret = dma_mapping_error(dev->dev, dsts[i]); 633 if (ret) { 634 dmaengine_unmap_put(um); 635 result("dst mapping error", total_tests, 636 src_off, dst_off, len, ret); 637 failed_tests++; 638 continue; 639 } 640 um->bidi_cnt++; 641 } 642 643 if (thread->type == DMA_MEMCPY) 644 tx = dev->device_prep_dma_memcpy(chan, 645 dsts[0] + dst_off, 646 srcs[0], len, flags); 647 else if (thread->type == DMA_MEMSET) 648 tx = dev->device_prep_dma_memset(chan, 649 dsts[0] + dst_off, 650 *(thread->srcs[0] + src_off), 651 len, flags); 652 else if (thread->type == DMA_XOR) 653 tx = dev->device_prep_dma_xor(chan, 654 dsts[0] + dst_off, 655 srcs, src_cnt, 656 len, flags); 657 else if (thread->type == DMA_PQ) { 658 dma_addr_t dma_pq[dst_cnt]; 659 660 for (i = 0; i < dst_cnt; i++) 661 dma_pq[i] = dsts[i] + dst_off; 662 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, 663 src_cnt, pq_coefs, 664 len, flags); 665 } 666 667 if (!tx) { 668 dmaengine_unmap_put(um); 669 result("prep error", total_tests, src_off, 670 dst_off, len, ret); 671 msleep(100); 672 failed_tests++; 673 continue; 674 } 675 676 done.done = false; 677 tx->callback = dmatest_callback; 678 tx->callback_param = &done; 679 cookie = tx->tx_submit(tx); 680 681 if (dma_submit_error(cookie)) { 682 dmaengine_unmap_put(um); 683 result("submit error", total_tests, src_off, 684 dst_off, len, ret); 685 msleep(100); 686 failed_tests++; 687 continue; 688 } 689 dma_async_issue_pending(chan); 690 691 wait_event_freezable_timeout(done_wait, done.done, 692 msecs_to_jiffies(params->timeout)); 693 694 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 695 696 if (!done.done) { 697 /* 698 * We're leaving the timed out dma operation with 699 * dangling pointer to done_wait. To make this 700 * correct, we'll need to allocate wait_done for 701 * each test iteration and perform "who's gonna 702 * free it this time?" dancing. For now, just 703 * leave it dangling. 704 */ 705 dmaengine_unmap_put(um); 706 result("test timed out", total_tests, src_off, dst_off, 707 len, 0); 708 failed_tests++; 709 continue; 710 } else if (status != DMA_COMPLETE) { 711 dmaengine_unmap_put(um); 712 result(status == DMA_ERROR ? 713 "completion error status" : 714 "completion busy status", total_tests, src_off, 715 dst_off, len, ret); 716 failed_tests++; 717 continue; 718 } 719 720 dmaengine_unmap_put(um); 721 722 if (params->noverify) { 723 verbose_result("test passed", total_tests, src_off, 724 dst_off, len, 0); 725 continue; 726 } 727 728 start = ktime_get(); 729 pr_debug("%s: verifying source buffer...\n", current->comm); 730 error_count = dmatest_verify(thread->srcs, 0, src_off, 731 0, PATTERN_SRC, true, is_memset); 732 error_count += dmatest_verify(thread->srcs, src_off, 733 src_off + len, src_off, 734 PATTERN_SRC | PATTERN_COPY, true, is_memset); 735 error_count += dmatest_verify(thread->srcs, src_off + len, 736 params->buf_size, src_off + len, 737 PATTERN_SRC, true, is_memset); 738 739 pr_debug("%s: verifying dest buffer...\n", current->comm); 740 error_count += dmatest_verify(thread->dsts, 0, dst_off, 741 0, PATTERN_DST, false, is_memset); 742 743 error_count += dmatest_verify(thread->dsts, dst_off, 744 dst_off + len, src_off, 745 PATTERN_SRC | PATTERN_COPY, false, is_memset); 746 747 error_count += dmatest_verify(thread->dsts, dst_off + len, 748 params->buf_size, dst_off + len, 749 PATTERN_DST, false, is_memset); 750 751 diff = ktime_sub(ktime_get(), start); 752 comparetime = ktime_add(comparetime, diff); 753 754 if (error_count) { 755 result("data error", total_tests, src_off, dst_off, 756 len, error_count); 757 failed_tests++; 758 } else { 759 verbose_result("test passed", total_tests, src_off, 760 dst_off, len, 0); 761 } 762 } 763 ktime = ktime_sub(ktime_get(), ktime); 764 ktime = ktime_sub(ktime, comparetime); 765 ktime = ktime_sub(ktime, filltime); 766 runtime = ktime_to_us(ktime); 767 768 ret = 0; 769 err_dstbuf: 770 for (i = 0; thread->udsts[i]; i++) 771 kfree(thread->udsts[i]); 772 kfree(thread->udsts); 773 err_udsts: 774 kfree(thread->dsts); 775 err_dsts: 776 err_srcbuf: 777 for (i = 0; thread->usrcs[i]; i++) 778 kfree(thread->usrcs[i]); 779 kfree(thread->usrcs); 780 err_usrcs: 781 kfree(thread->srcs); 782 err_srcs: 783 kfree(pq_coefs); 784 err_thread_type: 785 pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n", 786 current->comm, total_tests, failed_tests, 787 dmatest_persec(runtime, total_tests), 788 dmatest_KBs(runtime, total_len), ret); 789 790 /* terminate all transfers on specified channels */ 791 if (ret) 792 dmaengine_terminate_all(chan); 793 794 thread->done = true; 795 wake_up(&thread_wait); 796 797 return ret; 798 } 799 800 static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 801 { 802 struct dmatest_thread *thread; 803 struct dmatest_thread *_thread; 804 int ret; 805 806 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 807 ret = kthread_stop(thread->task); 808 pr_debug("thread %s exited with status %d\n", 809 thread->task->comm, ret); 810 list_del(&thread->node); 811 put_task_struct(thread->task); 812 kfree(thread); 813 } 814 815 /* terminate all transfers on specified channels */ 816 dmaengine_terminate_all(dtc->chan); 817 818 kfree(dtc); 819 } 820 821 static int dmatest_add_threads(struct dmatest_info *info, 822 struct dmatest_chan *dtc, enum dma_transaction_type type) 823 { 824 struct dmatest_params *params = &info->params; 825 struct dmatest_thread *thread; 826 struct dma_chan *chan = dtc->chan; 827 char *op; 828 unsigned int i; 829 830 if (type == DMA_MEMCPY) 831 op = "copy"; 832 else if (type == DMA_MEMSET) 833 op = "set"; 834 else if (type == DMA_XOR) 835 op = "xor"; 836 else if (type == DMA_PQ) 837 op = "pq"; 838 else 839 return -EINVAL; 840 841 for (i = 0; i < params->threads_per_chan; i++) { 842 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 843 if (!thread) { 844 pr_warn("No memory for %s-%s%u\n", 845 dma_chan_name(chan), op, i); 846 break; 847 } 848 thread->info = info; 849 thread->chan = dtc->chan; 850 thread->type = type; 851 smp_wmb(); 852 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u", 853 dma_chan_name(chan), op, i); 854 if (IS_ERR(thread->task)) { 855 pr_warn("Failed to create thread %s-%s%u\n", 856 dma_chan_name(chan), op, i); 857 kfree(thread); 858 break; 859 } 860 861 /* srcbuf and dstbuf are allocated by the thread itself */ 862 get_task_struct(thread->task); 863 list_add_tail(&thread->node, &dtc->threads); 864 wake_up_process(thread->task); 865 } 866 867 return i; 868 } 869 870 static int dmatest_add_channel(struct dmatest_info *info, 871 struct dma_chan *chan) 872 { 873 struct dmatest_chan *dtc; 874 struct dma_device *dma_dev = chan->device; 875 unsigned int thread_count = 0; 876 int cnt; 877 878 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 879 if (!dtc) { 880 pr_warn("No memory for %s\n", dma_chan_name(chan)); 881 return -ENOMEM; 882 } 883 884 dtc->chan = chan; 885 INIT_LIST_HEAD(&dtc->threads); 886 887 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 888 if (dmatest == 0) { 889 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY); 890 thread_count += cnt > 0 ? cnt : 0; 891 } 892 } 893 894 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) { 895 if (dmatest == 1) { 896 cnt = dmatest_add_threads(info, dtc, DMA_MEMSET); 897 thread_count += cnt > 0 ? cnt : 0; 898 } 899 } 900 901 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 902 cnt = dmatest_add_threads(info, dtc, DMA_XOR); 903 thread_count += cnt > 0 ? cnt : 0; 904 } 905 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 906 cnt = dmatest_add_threads(info, dtc, DMA_PQ); 907 thread_count += cnt > 0 ? cnt : 0; 908 } 909 910 pr_info("Started %u threads using %s\n", 911 thread_count, dma_chan_name(chan)); 912 913 list_add_tail(&dtc->node, &info->channels); 914 info->nr_channels++; 915 916 return 0; 917 } 918 919 static bool filter(struct dma_chan *chan, void *param) 920 { 921 struct dmatest_params *params = param; 922 923 if (!dmatest_match_channel(params, chan) || 924 !dmatest_match_device(params, chan->device)) 925 return false; 926 else 927 return true; 928 } 929 930 static void request_channels(struct dmatest_info *info, 931 enum dma_transaction_type type) 932 { 933 dma_cap_mask_t mask; 934 935 dma_cap_zero(mask); 936 dma_cap_set(type, mask); 937 for (;;) { 938 struct dmatest_params *params = &info->params; 939 struct dma_chan *chan; 940 941 chan = dma_request_channel(mask, filter, params); 942 if (chan) { 943 if (dmatest_add_channel(info, chan)) { 944 dma_release_channel(chan); 945 break; /* add_channel failed, punt */ 946 } 947 } else 948 break; /* no more channels available */ 949 if (params->max_channels && 950 info->nr_channels >= params->max_channels) 951 break; /* we have all we need */ 952 } 953 } 954 955 static void run_threaded_test(struct dmatest_info *info) 956 { 957 struct dmatest_params *params = &info->params; 958 959 /* Copy test parameters */ 960 params->buf_size = test_buf_size; 961 strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); 962 strlcpy(params->device, strim(test_device), sizeof(params->device)); 963 params->threads_per_chan = threads_per_chan; 964 params->max_channels = max_channels; 965 params->iterations = iterations; 966 params->xor_sources = xor_sources; 967 params->pq_sources = pq_sources; 968 params->timeout = timeout; 969 params->noverify = noverify; 970 971 request_channels(info, DMA_MEMCPY); 972 request_channels(info, DMA_MEMSET); 973 request_channels(info, DMA_XOR); 974 request_channels(info, DMA_PQ); 975 } 976 977 static void stop_threaded_test(struct dmatest_info *info) 978 { 979 struct dmatest_chan *dtc, *_dtc; 980 struct dma_chan *chan; 981 982 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { 983 list_del(&dtc->node); 984 chan = dtc->chan; 985 dmatest_cleanup_channel(dtc); 986 pr_debug("dropped channel %s\n", dma_chan_name(chan)); 987 dma_release_channel(chan); 988 } 989 990 info->nr_channels = 0; 991 } 992 993 static void restart_threaded_test(struct dmatest_info *info, bool run) 994 { 995 /* we might be called early to set run=, defer running until all 996 * parameters have been evaluated 997 */ 998 if (!info->did_init) 999 return; 1000 1001 /* Stop any running test first */ 1002 stop_threaded_test(info); 1003 1004 /* Run test with new parameters */ 1005 run_threaded_test(info); 1006 } 1007 1008 static int dmatest_run_get(char *val, const struct kernel_param *kp) 1009 { 1010 struct dmatest_info *info = &test_info; 1011 1012 mutex_lock(&info->lock); 1013 if (is_threaded_test_run(info)) { 1014 dmatest_run = true; 1015 } else { 1016 stop_threaded_test(info); 1017 dmatest_run = false; 1018 } 1019 mutex_unlock(&info->lock); 1020 1021 return param_get_bool(val, kp); 1022 } 1023 1024 static int dmatest_run_set(const char *val, const struct kernel_param *kp) 1025 { 1026 struct dmatest_info *info = &test_info; 1027 int ret; 1028 1029 mutex_lock(&info->lock); 1030 ret = param_set_bool(val, kp); 1031 if (ret) { 1032 mutex_unlock(&info->lock); 1033 return ret; 1034 } 1035 1036 if (is_threaded_test_run(info)) 1037 ret = -EBUSY; 1038 else if (dmatest_run) 1039 restart_threaded_test(info, dmatest_run); 1040 1041 mutex_unlock(&info->lock); 1042 1043 return ret; 1044 } 1045 1046 static int __init dmatest_init(void) 1047 { 1048 struct dmatest_info *info = &test_info; 1049 struct dmatest_params *params = &info->params; 1050 1051 if (dmatest_run) { 1052 mutex_lock(&info->lock); 1053 run_threaded_test(info); 1054 mutex_unlock(&info->lock); 1055 } 1056 1057 if (params->iterations && wait) 1058 wait_event(thread_wait, !is_threaded_test_run(info)); 1059 1060 /* module parameters are stable, inittime tests are started, 1061 * let userspace take over 'run' control 1062 */ 1063 info->did_init = true; 1064 1065 return 0; 1066 } 1067 /* when compiled-in wait for drivers to load first */ 1068 late_initcall(dmatest_init); 1069 1070 static void __exit dmatest_exit(void) 1071 { 1072 struct dmatest_info *info = &test_info; 1073 1074 mutex_lock(&info->lock); 1075 stop_threaded_test(info); 1076 mutex_unlock(&info->lock); 1077 } 1078 module_exit(dmatest_exit); 1079 1080 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 1081 MODULE_LICENSE("GPL v2"); 1082