1 /* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * Copyright (C) 2013 Intel Corporation 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/delay.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/dmaengine.h> 16 #include <linux/freezer.h> 17 #include <linux/init.h> 18 #include <linux/kthread.h> 19 #include <linux/module.h> 20 #include <linux/moduleparam.h> 21 #include <linux/random.h> 22 #include <linux/slab.h> 23 #include <linux/wait.h> 24 25 static unsigned int test_buf_size = 16384; 26 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR); 27 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 28 29 static char test_channel[20]; 30 module_param_string(channel, test_channel, sizeof(test_channel), 31 S_IRUGO | S_IWUSR); 32 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 33 34 static char test_device[32]; 35 module_param_string(device, test_device, sizeof(test_device), 36 S_IRUGO | S_IWUSR); 37 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 38 39 static unsigned int threads_per_chan = 1; 40 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR); 41 MODULE_PARM_DESC(threads_per_chan, 42 "Number of threads to start per channel (default: 1)"); 43 44 static unsigned int max_channels; 45 module_param(max_channels, uint, S_IRUGO | S_IWUSR); 46 MODULE_PARM_DESC(max_channels, 47 "Maximum number of channels to use (default: all)"); 48 49 static unsigned int iterations; 50 module_param(iterations, uint, S_IRUGO | S_IWUSR); 51 MODULE_PARM_DESC(iterations, 52 "Iterations before stopping test (default: infinite)"); 53 54 static unsigned int sg_buffers = 1; 55 module_param(sg_buffers, uint, S_IRUGO | S_IWUSR); 56 MODULE_PARM_DESC(sg_buffers, 57 "Number of scatter gather buffers (default: 1)"); 58 59 static unsigned int dmatest; 60 module_param(dmatest, uint, S_IRUGO | S_IWUSR); 61 MODULE_PARM_DESC(dmatest, 62 "dmatest 0-memcpy 1-slave_sg (default: 0)"); 63 64 static unsigned int xor_sources = 3; 65 module_param(xor_sources, uint, S_IRUGO | S_IWUSR); 66 MODULE_PARM_DESC(xor_sources, 67 "Number of xor source buffers (default: 3)"); 68 69 static unsigned int pq_sources = 3; 70 module_param(pq_sources, uint, S_IRUGO | S_IWUSR); 71 MODULE_PARM_DESC(pq_sources, 72 "Number of p+q source buffers (default: 3)"); 73 74 static int timeout = 3000; 75 module_param(timeout, uint, S_IRUGO | S_IWUSR); 76 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 77 "Pass -1 for infinite timeout"); 78 79 static bool noverify; 80 module_param(noverify, bool, S_IRUGO | S_IWUSR); 81 MODULE_PARM_DESC(noverify, "Disable random data setup and verification"); 82 83 static bool verbose; 84 module_param(verbose, bool, S_IRUGO | S_IWUSR); 85 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)"); 86 87 /** 88 * struct dmatest_params - test parameters. 89 * @buf_size: size of the memcpy test buffer 90 * @channel: bus ID of the channel to test 91 * @device: bus ID of the DMA Engine to test 92 * @threads_per_chan: number of threads to start per channel 93 * @max_channels: maximum number of channels to use 94 * @iterations: iterations before stopping test 95 * @xor_sources: number of xor source buffers 96 * @pq_sources: number of p+q source buffers 97 * @timeout: transfer timeout in msec, -1 for infinite timeout 98 */ 99 struct dmatest_params { 100 unsigned int buf_size; 101 char channel[20]; 102 char device[32]; 103 unsigned int threads_per_chan; 104 unsigned int max_channels; 105 unsigned int iterations; 106 unsigned int xor_sources; 107 unsigned int pq_sources; 108 int timeout; 109 bool noverify; 110 }; 111 112 /** 113 * struct dmatest_info - test information. 114 * @params: test parameters 115 * @lock: access protection to the fields of this structure 116 */ 117 static struct dmatest_info { 118 /* Test parameters */ 119 struct dmatest_params params; 120 121 /* Internal state */ 122 struct list_head channels; 123 unsigned int nr_channels; 124 struct mutex lock; 125 bool did_init; 126 } test_info = { 127 .channels = LIST_HEAD_INIT(test_info.channels), 128 .lock = __MUTEX_INITIALIZER(test_info.lock), 129 }; 130 131 static int dmatest_run_set(const char *val, const struct kernel_param *kp); 132 static int dmatest_run_get(char *val, const struct kernel_param *kp); 133 static const struct kernel_param_ops run_ops = { 134 .set = dmatest_run_set, 135 .get = dmatest_run_get, 136 }; 137 static bool dmatest_run; 138 module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR); 139 MODULE_PARM_DESC(run, "Run the test (default: false)"); 140 141 /* Maximum amount of mismatched bytes in buffer to print */ 142 #define MAX_ERROR_COUNT 32 143 144 /* 145 * Initialization patterns. All bytes in the source buffer has bit 7 146 * set, all bytes in the destination buffer has bit 7 cleared. 147 * 148 * Bit 6 is set for all bytes which are to be copied by the DMA 149 * engine. Bit 5 is set for all bytes which are to be overwritten by 150 * the DMA engine. 151 * 152 * The remaining bits are the inverse of a counter which increments by 153 * one for each byte address. 154 */ 155 #define PATTERN_SRC 0x80 156 #define PATTERN_DST 0x00 157 #define PATTERN_COPY 0x40 158 #define PATTERN_OVERWRITE 0x20 159 #define PATTERN_COUNT_MASK 0x1f 160 161 struct dmatest_thread { 162 struct list_head node; 163 struct dmatest_info *info; 164 struct task_struct *task; 165 struct dma_chan *chan; 166 u8 **srcs; 167 u8 **usrcs; 168 u8 **dsts; 169 u8 **udsts; 170 enum dma_transaction_type type; 171 bool done; 172 }; 173 174 struct dmatest_chan { 175 struct list_head node; 176 struct dma_chan *chan; 177 struct list_head threads; 178 }; 179 180 static DECLARE_WAIT_QUEUE_HEAD(thread_wait); 181 static bool wait; 182 183 static bool is_threaded_test_run(struct dmatest_info *info) 184 { 185 struct dmatest_chan *dtc; 186 187 list_for_each_entry(dtc, &info->channels, node) { 188 struct dmatest_thread *thread; 189 190 list_for_each_entry(thread, &dtc->threads, node) { 191 if (!thread->done) 192 return true; 193 } 194 } 195 196 return false; 197 } 198 199 static int dmatest_wait_get(char *val, const struct kernel_param *kp) 200 { 201 struct dmatest_info *info = &test_info; 202 struct dmatest_params *params = &info->params; 203 204 if (params->iterations) 205 wait_event(thread_wait, !is_threaded_test_run(info)); 206 wait = true; 207 return param_get_bool(val, kp); 208 } 209 210 static const struct kernel_param_ops wait_ops = { 211 .get = dmatest_wait_get, 212 .set = param_set_bool, 213 }; 214 module_param_cb(wait, &wait_ops, &wait, S_IRUGO); 215 MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)"); 216 217 static bool dmatest_match_channel(struct dmatest_params *params, 218 struct dma_chan *chan) 219 { 220 if (params->channel[0] == '\0') 221 return true; 222 return strcmp(dma_chan_name(chan), params->channel) == 0; 223 } 224 225 static bool dmatest_match_device(struct dmatest_params *params, 226 struct dma_device *device) 227 { 228 if (params->device[0] == '\0') 229 return true; 230 return strcmp(dev_name(device->dev), params->device) == 0; 231 } 232 233 static unsigned long dmatest_random(void) 234 { 235 unsigned long buf; 236 237 prandom_bytes(&buf, sizeof(buf)); 238 return buf; 239 } 240 241 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len, 242 unsigned int buf_size) 243 { 244 unsigned int i; 245 u8 *buf; 246 247 for (; (buf = *bufs); bufs++) { 248 for (i = 0; i < start; i++) 249 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 250 for ( ; i < start + len; i++) 251 buf[i] = PATTERN_SRC | PATTERN_COPY 252 | (~i & PATTERN_COUNT_MASK); 253 for ( ; i < buf_size; i++) 254 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 255 buf++; 256 } 257 } 258 259 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len, 260 unsigned int buf_size) 261 { 262 unsigned int i; 263 u8 *buf; 264 265 for (; (buf = *bufs); bufs++) { 266 for (i = 0; i < start; i++) 267 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 268 for ( ; i < start + len; i++) 269 buf[i] = PATTERN_DST | PATTERN_OVERWRITE 270 | (~i & PATTERN_COUNT_MASK); 271 for ( ; i < buf_size; i++) 272 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 273 } 274 } 275 276 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 277 unsigned int counter, bool is_srcbuf) 278 { 279 u8 diff = actual ^ pattern; 280 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 281 const char *thread_name = current->comm; 282 283 if (is_srcbuf) 284 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n", 285 thread_name, index, expected, actual); 286 else if ((pattern & PATTERN_COPY) 287 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 288 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n", 289 thread_name, index, expected, actual); 290 else if (diff & PATTERN_SRC) 291 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n", 292 thread_name, index, expected, actual); 293 else 294 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n", 295 thread_name, index, expected, actual); 296 } 297 298 static unsigned int dmatest_verify(u8 **bufs, unsigned int start, 299 unsigned int end, unsigned int counter, u8 pattern, 300 bool is_srcbuf) 301 { 302 unsigned int i; 303 unsigned int error_count = 0; 304 u8 actual; 305 u8 expected; 306 u8 *buf; 307 unsigned int counter_orig = counter; 308 309 for (; (buf = *bufs); bufs++) { 310 counter = counter_orig; 311 for (i = start; i < end; i++) { 312 actual = buf[i]; 313 expected = pattern | (~counter & PATTERN_COUNT_MASK); 314 if (actual != expected) { 315 if (error_count < MAX_ERROR_COUNT) 316 dmatest_mismatch(actual, pattern, i, 317 counter, is_srcbuf); 318 error_count++; 319 } 320 counter++; 321 } 322 } 323 324 if (error_count > MAX_ERROR_COUNT) 325 pr_warn("%s: %u errors suppressed\n", 326 current->comm, error_count - MAX_ERROR_COUNT); 327 328 return error_count; 329 } 330 331 /* poor man's completion - we want to use wait_event_freezable() on it */ 332 struct dmatest_done { 333 bool done; 334 wait_queue_head_t *wait; 335 }; 336 337 static void dmatest_callback(void *arg) 338 { 339 struct dmatest_done *done = arg; 340 341 done->done = true; 342 wake_up_all(done->wait); 343 } 344 345 static unsigned int min_odd(unsigned int x, unsigned int y) 346 { 347 unsigned int val = min(x, y); 348 349 return val % 2 ? val : val - 1; 350 } 351 352 static void result(const char *err, unsigned int n, unsigned int src_off, 353 unsigned int dst_off, unsigned int len, unsigned long data) 354 { 355 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n", 356 current->comm, n, err, src_off, dst_off, len, data); 357 } 358 359 static void dbg_result(const char *err, unsigned int n, unsigned int src_off, 360 unsigned int dst_off, unsigned int len, 361 unsigned long data) 362 { 363 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n", 364 current->comm, n, err, src_off, dst_off, len, data); 365 } 366 367 #define verbose_result(err, n, src_off, dst_off, len, data) ({ \ 368 if (verbose) \ 369 result(err, n, src_off, dst_off, len, data); \ 370 else \ 371 dbg_result(err, n, src_off, dst_off, len, data);\ 372 }) 373 374 static unsigned long long dmatest_persec(s64 runtime, unsigned int val) 375 { 376 unsigned long long per_sec = 1000000; 377 378 if (runtime <= 0) 379 return 0; 380 381 /* drop precision until runtime is 32-bits */ 382 while (runtime > UINT_MAX) { 383 runtime >>= 1; 384 per_sec <<= 1; 385 } 386 387 per_sec *= val; 388 do_div(per_sec, runtime); 389 return per_sec; 390 } 391 392 static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len) 393 { 394 return dmatest_persec(runtime, len >> 10); 395 } 396 397 /* 398 * This function repeatedly tests DMA transfers of various lengths and 399 * offsets for a given operation type until it is told to exit by 400 * kthread_stop(). There may be multiple threads running this function 401 * in parallel for a single channel, and there may be multiple channels 402 * being tested in parallel. 403 * 404 * Before each test, the source and destination buffer is initialized 405 * with a known pattern. This pattern is different depending on 406 * whether it's in an area which is supposed to be copied or 407 * overwritten, and different in the source and destination buffers. 408 * So if the DMA engine doesn't copy exactly what we tell it to copy, 409 * we'll notice. 410 */ 411 static int dmatest_func(void *data) 412 { 413 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait); 414 struct dmatest_thread *thread = data; 415 struct dmatest_done done = { .wait = &done_wait }; 416 struct dmatest_info *info; 417 struct dmatest_params *params; 418 struct dma_chan *chan; 419 struct dma_device *dev; 420 unsigned int error_count; 421 unsigned int failed_tests = 0; 422 unsigned int total_tests = 0; 423 dma_cookie_t cookie; 424 enum dma_status status; 425 enum dma_ctrl_flags flags; 426 u8 *pq_coefs = NULL; 427 int ret; 428 int src_cnt; 429 int dst_cnt; 430 int i; 431 ktime_t ktime, start, diff; 432 ktime_t filltime = 0; 433 ktime_t comparetime = 0; 434 s64 runtime = 0; 435 unsigned long long total_len = 0; 436 u8 align = 0; 437 438 set_freezable(); 439 440 ret = -ENOMEM; 441 442 smp_rmb(); 443 info = thread->info; 444 params = &info->params; 445 chan = thread->chan; 446 dev = chan->device; 447 if (thread->type == DMA_MEMCPY) { 448 align = dev->copy_align; 449 src_cnt = dst_cnt = 1; 450 } else if (thread->type == DMA_SG) { 451 align = dev->copy_align; 452 src_cnt = dst_cnt = sg_buffers; 453 } else if (thread->type == DMA_XOR) { 454 /* force odd to ensure dst = src */ 455 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor); 456 dst_cnt = 1; 457 align = dev->xor_align; 458 } else if (thread->type == DMA_PQ) { 459 /* force odd to ensure dst = src */ 460 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); 461 dst_cnt = 2; 462 align = dev->pq_align; 463 464 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL); 465 if (!pq_coefs) 466 goto err_thread_type; 467 468 for (i = 0; i < src_cnt; i++) 469 pq_coefs[i] = 1; 470 } else 471 goto err_thread_type; 472 473 thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL); 474 if (!thread->srcs) 475 goto err_srcs; 476 477 thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL); 478 if (!thread->usrcs) 479 goto err_usrcs; 480 481 for (i = 0; i < src_cnt; i++) { 482 thread->usrcs[i] = kmalloc(params->buf_size + align, 483 GFP_KERNEL); 484 if (!thread->usrcs[i]) 485 goto err_srcbuf; 486 487 /* align srcs to alignment restriction */ 488 if (align) 489 thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align); 490 else 491 thread->srcs[i] = thread->usrcs[i]; 492 } 493 thread->srcs[i] = NULL; 494 495 thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL); 496 if (!thread->dsts) 497 goto err_dsts; 498 499 thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL); 500 if (!thread->udsts) 501 goto err_udsts; 502 503 for (i = 0; i < dst_cnt; i++) { 504 thread->udsts[i] = kmalloc(params->buf_size + align, 505 GFP_KERNEL); 506 if (!thread->udsts[i]) 507 goto err_dstbuf; 508 509 /* align dsts to alignment restriction */ 510 if (align) 511 thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align); 512 else 513 thread->dsts[i] = thread->udsts[i]; 514 } 515 thread->dsts[i] = NULL; 516 517 set_user_nice(current, 10); 518 519 /* 520 * src and dst buffers are freed by ourselves below 521 */ 522 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; 523 524 ktime = ktime_get(); 525 while (!kthread_should_stop() 526 && !(params->iterations && total_tests >= params->iterations)) { 527 struct dma_async_tx_descriptor *tx = NULL; 528 struct dmaengine_unmap_data *um; 529 dma_addr_t srcs[src_cnt]; 530 dma_addr_t *dsts; 531 unsigned int src_off, dst_off, len; 532 struct scatterlist tx_sg[src_cnt]; 533 struct scatterlist rx_sg[src_cnt]; 534 535 total_tests++; 536 537 if (1 << align > params->buf_size) { 538 pr_err("%u-byte buffer too small for %d-byte alignment\n", 539 params->buf_size, 1 << align); 540 break; 541 } 542 543 if (params->noverify) 544 len = params->buf_size; 545 else 546 len = dmatest_random() % params->buf_size + 1; 547 548 len = (len >> align) << align; 549 if (!len) 550 len = 1 << align; 551 552 total_len += len; 553 554 if (params->noverify) { 555 src_off = 0; 556 dst_off = 0; 557 } else { 558 start = ktime_get(); 559 src_off = dmatest_random() % (params->buf_size - len + 1); 560 dst_off = dmatest_random() % (params->buf_size - len + 1); 561 562 src_off = (src_off >> align) << align; 563 dst_off = (dst_off >> align) << align; 564 565 dmatest_init_srcs(thread->srcs, src_off, len, 566 params->buf_size); 567 dmatest_init_dsts(thread->dsts, dst_off, len, 568 params->buf_size); 569 570 diff = ktime_sub(ktime_get(), start); 571 filltime = ktime_add(filltime, diff); 572 } 573 574 um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt, 575 GFP_KERNEL); 576 if (!um) { 577 failed_tests++; 578 result("unmap data NULL", total_tests, 579 src_off, dst_off, len, ret); 580 continue; 581 } 582 583 um->len = params->buf_size; 584 for (i = 0; i < src_cnt; i++) { 585 void *buf = thread->srcs[i]; 586 struct page *pg = virt_to_page(buf); 587 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK; 588 589 um->addr[i] = dma_map_page(dev->dev, pg, pg_off, 590 um->len, DMA_TO_DEVICE); 591 srcs[i] = um->addr[i] + src_off; 592 ret = dma_mapping_error(dev->dev, um->addr[i]); 593 if (ret) { 594 dmaengine_unmap_put(um); 595 result("src mapping error", total_tests, 596 src_off, dst_off, len, ret); 597 failed_tests++; 598 continue; 599 } 600 um->to_cnt++; 601 } 602 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 603 dsts = &um->addr[src_cnt]; 604 for (i = 0; i < dst_cnt; i++) { 605 void *buf = thread->dsts[i]; 606 struct page *pg = virt_to_page(buf); 607 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK; 608 609 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len, 610 DMA_BIDIRECTIONAL); 611 ret = dma_mapping_error(dev->dev, dsts[i]); 612 if (ret) { 613 dmaengine_unmap_put(um); 614 result("dst mapping error", total_tests, 615 src_off, dst_off, len, ret); 616 failed_tests++; 617 continue; 618 } 619 um->bidi_cnt++; 620 } 621 622 sg_init_table(tx_sg, src_cnt); 623 sg_init_table(rx_sg, src_cnt); 624 for (i = 0; i < src_cnt; i++) { 625 sg_dma_address(&rx_sg[i]) = srcs[i]; 626 sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off; 627 sg_dma_len(&tx_sg[i]) = len; 628 sg_dma_len(&rx_sg[i]) = len; 629 } 630 631 if (thread->type == DMA_MEMCPY) 632 tx = dev->device_prep_dma_memcpy(chan, 633 dsts[0] + dst_off, 634 srcs[0], len, flags); 635 else if (thread->type == DMA_SG) 636 tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt, 637 rx_sg, src_cnt, flags); 638 else if (thread->type == DMA_XOR) 639 tx = dev->device_prep_dma_xor(chan, 640 dsts[0] + dst_off, 641 srcs, src_cnt, 642 len, flags); 643 else if (thread->type == DMA_PQ) { 644 dma_addr_t dma_pq[dst_cnt]; 645 646 for (i = 0; i < dst_cnt; i++) 647 dma_pq[i] = dsts[i] + dst_off; 648 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, 649 src_cnt, pq_coefs, 650 len, flags); 651 } 652 653 if (!tx) { 654 dmaengine_unmap_put(um); 655 result("prep error", total_tests, src_off, 656 dst_off, len, ret); 657 msleep(100); 658 failed_tests++; 659 continue; 660 } 661 662 done.done = false; 663 tx->callback = dmatest_callback; 664 tx->callback_param = &done; 665 cookie = tx->tx_submit(tx); 666 667 if (dma_submit_error(cookie)) { 668 dmaengine_unmap_put(um); 669 result("submit error", total_tests, src_off, 670 dst_off, len, ret); 671 msleep(100); 672 failed_tests++; 673 continue; 674 } 675 dma_async_issue_pending(chan); 676 677 wait_event_freezable_timeout(done_wait, done.done, 678 msecs_to_jiffies(params->timeout)); 679 680 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 681 682 if (!done.done) { 683 /* 684 * We're leaving the timed out dma operation with 685 * dangling pointer to done_wait. To make this 686 * correct, we'll need to allocate wait_done for 687 * each test iteration and perform "who's gonna 688 * free it this time?" dancing. For now, just 689 * leave it dangling. 690 */ 691 dmaengine_unmap_put(um); 692 result("test timed out", total_tests, src_off, dst_off, 693 len, 0); 694 failed_tests++; 695 continue; 696 } else if (status != DMA_COMPLETE) { 697 dmaengine_unmap_put(um); 698 result(status == DMA_ERROR ? 699 "completion error status" : 700 "completion busy status", total_tests, src_off, 701 dst_off, len, ret); 702 failed_tests++; 703 continue; 704 } 705 706 dmaengine_unmap_put(um); 707 708 if (params->noverify) { 709 verbose_result("test passed", total_tests, src_off, 710 dst_off, len, 0); 711 continue; 712 } 713 714 start = ktime_get(); 715 pr_debug("%s: verifying source buffer...\n", current->comm); 716 error_count = dmatest_verify(thread->srcs, 0, src_off, 717 0, PATTERN_SRC, true); 718 error_count += dmatest_verify(thread->srcs, src_off, 719 src_off + len, src_off, 720 PATTERN_SRC | PATTERN_COPY, true); 721 error_count += dmatest_verify(thread->srcs, src_off + len, 722 params->buf_size, src_off + len, 723 PATTERN_SRC, true); 724 725 pr_debug("%s: verifying dest buffer...\n", current->comm); 726 error_count += dmatest_verify(thread->dsts, 0, dst_off, 727 0, PATTERN_DST, false); 728 error_count += dmatest_verify(thread->dsts, dst_off, 729 dst_off + len, src_off, 730 PATTERN_SRC | PATTERN_COPY, false); 731 error_count += dmatest_verify(thread->dsts, dst_off + len, 732 params->buf_size, dst_off + len, 733 PATTERN_DST, false); 734 735 diff = ktime_sub(ktime_get(), start); 736 comparetime = ktime_add(comparetime, diff); 737 738 if (error_count) { 739 result("data error", total_tests, src_off, dst_off, 740 len, error_count); 741 failed_tests++; 742 } else { 743 verbose_result("test passed", total_tests, src_off, 744 dst_off, len, 0); 745 } 746 } 747 ktime = ktime_sub(ktime_get(), ktime); 748 ktime = ktime_sub(ktime, comparetime); 749 ktime = ktime_sub(ktime, filltime); 750 runtime = ktime_to_us(ktime); 751 752 ret = 0; 753 err_dstbuf: 754 for (i = 0; thread->udsts[i]; i++) 755 kfree(thread->udsts[i]); 756 kfree(thread->udsts); 757 err_udsts: 758 kfree(thread->dsts); 759 err_dsts: 760 err_srcbuf: 761 for (i = 0; thread->usrcs[i]; i++) 762 kfree(thread->usrcs[i]); 763 kfree(thread->usrcs); 764 err_usrcs: 765 kfree(thread->srcs); 766 err_srcs: 767 kfree(pq_coefs); 768 err_thread_type: 769 pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n", 770 current->comm, total_tests, failed_tests, 771 dmatest_persec(runtime, total_tests), 772 dmatest_KBs(runtime, total_len), ret); 773 774 /* terminate all transfers on specified channels */ 775 if (ret) 776 dmaengine_terminate_all(chan); 777 778 thread->done = true; 779 wake_up(&thread_wait); 780 781 return ret; 782 } 783 784 static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 785 { 786 struct dmatest_thread *thread; 787 struct dmatest_thread *_thread; 788 int ret; 789 790 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 791 ret = kthread_stop(thread->task); 792 pr_debug("thread %s exited with status %d\n", 793 thread->task->comm, ret); 794 list_del(&thread->node); 795 put_task_struct(thread->task); 796 kfree(thread); 797 } 798 799 /* terminate all transfers on specified channels */ 800 dmaengine_terminate_all(dtc->chan); 801 802 kfree(dtc); 803 } 804 805 static int dmatest_add_threads(struct dmatest_info *info, 806 struct dmatest_chan *dtc, enum dma_transaction_type type) 807 { 808 struct dmatest_params *params = &info->params; 809 struct dmatest_thread *thread; 810 struct dma_chan *chan = dtc->chan; 811 char *op; 812 unsigned int i; 813 814 if (type == DMA_MEMCPY) 815 op = "copy"; 816 else if (type == DMA_SG) 817 op = "sg"; 818 else if (type == DMA_XOR) 819 op = "xor"; 820 else if (type == DMA_PQ) 821 op = "pq"; 822 else 823 return -EINVAL; 824 825 for (i = 0; i < params->threads_per_chan; i++) { 826 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 827 if (!thread) { 828 pr_warn("No memory for %s-%s%u\n", 829 dma_chan_name(chan), op, i); 830 break; 831 } 832 thread->info = info; 833 thread->chan = dtc->chan; 834 thread->type = type; 835 smp_wmb(); 836 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u", 837 dma_chan_name(chan), op, i); 838 if (IS_ERR(thread->task)) { 839 pr_warn("Failed to create thread %s-%s%u\n", 840 dma_chan_name(chan), op, i); 841 kfree(thread); 842 break; 843 } 844 845 /* srcbuf and dstbuf are allocated by the thread itself */ 846 get_task_struct(thread->task); 847 list_add_tail(&thread->node, &dtc->threads); 848 wake_up_process(thread->task); 849 } 850 851 return i; 852 } 853 854 static int dmatest_add_channel(struct dmatest_info *info, 855 struct dma_chan *chan) 856 { 857 struct dmatest_chan *dtc; 858 struct dma_device *dma_dev = chan->device; 859 unsigned int thread_count = 0; 860 int cnt; 861 862 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 863 if (!dtc) { 864 pr_warn("No memory for %s\n", dma_chan_name(chan)); 865 return -ENOMEM; 866 } 867 868 dtc->chan = chan; 869 INIT_LIST_HEAD(&dtc->threads); 870 871 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 872 if (dmatest == 0) { 873 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY); 874 thread_count += cnt > 0 ? cnt : 0; 875 } 876 } 877 878 if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) { 879 if (dmatest == 1) { 880 cnt = dmatest_add_threads(info, dtc, DMA_SG); 881 thread_count += cnt > 0 ? cnt : 0; 882 } 883 } 884 885 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 886 cnt = dmatest_add_threads(info, dtc, DMA_XOR); 887 thread_count += cnt > 0 ? cnt : 0; 888 } 889 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 890 cnt = dmatest_add_threads(info, dtc, DMA_PQ); 891 thread_count += cnt > 0 ? cnt : 0; 892 } 893 894 pr_info("Started %u threads using %s\n", 895 thread_count, dma_chan_name(chan)); 896 897 list_add_tail(&dtc->node, &info->channels); 898 info->nr_channels++; 899 900 return 0; 901 } 902 903 static bool filter(struct dma_chan *chan, void *param) 904 { 905 struct dmatest_params *params = param; 906 907 if (!dmatest_match_channel(params, chan) || 908 !dmatest_match_device(params, chan->device)) 909 return false; 910 else 911 return true; 912 } 913 914 static void request_channels(struct dmatest_info *info, 915 enum dma_transaction_type type) 916 { 917 dma_cap_mask_t mask; 918 919 dma_cap_zero(mask); 920 dma_cap_set(type, mask); 921 for (;;) { 922 struct dmatest_params *params = &info->params; 923 struct dma_chan *chan; 924 925 chan = dma_request_channel(mask, filter, params); 926 if (chan) { 927 if (dmatest_add_channel(info, chan)) { 928 dma_release_channel(chan); 929 break; /* add_channel failed, punt */ 930 } 931 } else 932 break; /* no more channels available */ 933 if (params->max_channels && 934 info->nr_channels >= params->max_channels) 935 break; /* we have all we need */ 936 } 937 } 938 939 static void run_threaded_test(struct dmatest_info *info) 940 { 941 struct dmatest_params *params = &info->params; 942 943 /* Copy test parameters */ 944 params->buf_size = test_buf_size; 945 strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); 946 strlcpy(params->device, strim(test_device), sizeof(params->device)); 947 params->threads_per_chan = threads_per_chan; 948 params->max_channels = max_channels; 949 params->iterations = iterations; 950 params->xor_sources = xor_sources; 951 params->pq_sources = pq_sources; 952 params->timeout = timeout; 953 params->noverify = noverify; 954 955 request_channels(info, DMA_MEMCPY); 956 request_channels(info, DMA_XOR); 957 request_channels(info, DMA_SG); 958 request_channels(info, DMA_PQ); 959 } 960 961 static void stop_threaded_test(struct dmatest_info *info) 962 { 963 struct dmatest_chan *dtc, *_dtc; 964 struct dma_chan *chan; 965 966 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { 967 list_del(&dtc->node); 968 chan = dtc->chan; 969 dmatest_cleanup_channel(dtc); 970 pr_debug("dropped channel %s\n", dma_chan_name(chan)); 971 dma_release_channel(chan); 972 } 973 974 info->nr_channels = 0; 975 } 976 977 static void restart_threaded_test(struct dmatest_info *info, bool run) 978 { 979 /* we might be called early to set run=, defer running until all 980 * parameters have been evaluated 981 */ 982 if (!info->did_init) 983 return; 984 985 /* Stop any running test first */ 986 stop_threaded_test(info); 987 988 /* Run test with new parameters */ 989 run_threaded_test(info); 990 } 991 992 static int dmatest_run_get(char *val, const struct kernel_param *kp) 993 { 994 struct dmatest_info *info = &test_info; 995 996 mutex_lock(&info->lock); 997 if (is_threaded_test_run(info)) { 998 dmatest_run = true; 999 } else { 1000 stop_threaded_test(info); 1001 dmatest_run = false; 1002 } 1003 mutex_unlock(&info->lock); 1004 1005 return param_get_bool(val, kp); 1006 } 1007 1008 static int dmatest_run_set(const char *val, const struct kernel_param *kp) 1009 { 1010 struct dmatest_info *info = &test_info; 1011 int ret; 1012 1013 mutex_lock(&info->lock); 1014 ret = param_set_bool(val, kp); 1015 if (ret) { 1016 mutex_unlock(&info->lock); 1017 return ret; 1018 } 1019 1020 if (is_threaded_test_run(info)) 1021 ret = -EBUSY; 1022 else if (dmatest_run) 1023 restart_threaded_test(info, dmatest_run); 1024 1025 mutex_unlock(&info->lock); 1026 1027 return ret; 1028 } 1029 1030 static int __init dmatest_init(void) 1031 { 1032 struct dmatest_info *info = &test_info; 1033 struct dmatest_params *params = &info->params; 1034 1035 if (dmatest_run) { 1036 mutex_lock(&info->lock); 1037 run_threaded_test(info); 1038 mutex_unlock(&info->lock); 1039 } 1040 1041 if (params->iterations && wait) 1042 wait_event(thread_wait, !is_threaded_test_run(info)); 1043 1044 /* module parameters are stable, inittime tests are started, 1045 * let userspace take over 'run' control 1046 */ 1047 info->did_init = true; 1048 1049 return 0; 1050 } 1051 /* when compiled-in wait for drivers to load first */ 1052 late_initcall(dmatest_init); 1053 1054 static void __exit dmatest_exit(void) 1055 { 1056 struct dmatest_info *info = &test_info; 1057 1058 mutex_lock(&info->lock); 1059 stop_threaded_test(info); 1060 mutex_unlock(&info->lock); 1061 } 1062 module_exit(dmatest_exit); 1063 1064 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 1065 MODULE_LICENSE("GPL v2"); 1066