1 /* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/delay.h> 11 #include <linux/dmaengine.h> 12 #include <linux/init.h> 13 #include <linux/kthread.h> 14 #include <linux/module.h> 15 #include <linux/moduleparam.h> 16 #include <linux/random.h> 17 #include <linux/wait.h> 18 19 static unsigned int test_buf_size = 16384; 20 module_param(test_buf_size, uint, S_IRUGO); 21 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 22 23 static char test_channel[20]; 24 module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO); 25 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 26 27 static char test_device[20]; 28 module_param_string(device, test_device, sizeof(test_device), S_IRUGO); 29 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 30 31 static unsigned int threads_per_chan = 1; 32 module_param(threads_per_chan, uint, S_IRUGO); 33 MODULE_PARM_DESC(threads_per_chan, 34 "Number of threads to start per channel (default: 1)"); 35 36 static unsigned int max_channels; 37 module_param(max_channels, uint, S_IRUGO); 38 MODULE_PARM_DESC(max_channels, 39 "Maximum number of channels to use (default: all)"); 40 41 static unsigned int xor_sources = 3; 42 module_param(xor_sources, uint, S_IRUGO); 43 MODULE_PARM_DESC(xor_sources, 44 "Number of xor source buffers (default: 3)"); 45 46 /* 47 * Initialization patterns. All bytes in the source buffer has bit 7 48 * set, all bytes in the destination buffer has bit 7 cleared. 49 * 50 * Bit 6 is set for all bytes which are to be copied by the DMA 51 * engine. Bit 5 is set for all bytes which are to be overwritten by 52 * the DMA engine. 53 * 54 * The remaining bits are the inverse of a counter which increments by 55 * one for each byte address. 56 */ 57 #define PATTERN_SRC 0x80 58 #define PATTERN_DST 0x00 59 #define PATTERN_COPY 0x40 60 #define PATTERN_OVERWRITE 0x20 61 #define PATTERN_COUNT_MASK 0x1f 62 63 struct dmatest_thread { 64 struct list_head node; 65 struct task_struct *task; 66 struct dma_chan *chan; 67 u8 **srcs; 68 u8 **dsts; 69 enum dma_transaction_type type; 70 }; 71 72 struct dmatest_chan { 73 struct list_head node; 74 struct dma_chan *chan; 75 struct list_head threads; 76 }; 77 78 /* 79 * These are protected by dma_list_mutex since they're only used by 80 * the DMA filter function callback 81 */ 82 static LIST_HEAD(dmatest_channels); 83 static unsigned int nr_channels; 84 85 static bool dmatest_match_channel(struct dma_chan *chan) 86 { 87 if (test_channel[0] == '\0') 88 return true; 89 return strcmp(dma_chan_name(chan), test_channel) == 0; 90 } 91 92 static bool dmatest_match_device(struct dma_device *device) 93 { 94 if (test_device[0] == '\0') 95 return true; 96 return strcmp(dev_name(device->dev), test_device) == 0; 97 } 98 99 static unsigned long dmatest_random(void) 100 { 101 unsigned long buf; 102 103 get_random_bytes(&buf, sizeof(buf)); 104 return buf; 105 } 106 107 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len) 108 { 109 unsigned int i; 110 u8 *buf; 111 112 for (; (buf = *bufs); bufs++) { 113 for (i = 0; i < start; i++) 114 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 115 for ( ; i < start + len; i++) 116 buf[i] = PATTERN_SRC | PATTERN_COPY 117 | (~i & PATTERN_COUNT_MASK);; 118 for ( ; i < test_buf_size; i++) 119 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 120 buf++; 121 } 122 } 123 124 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len) 125 { 126 unsigned int i; 127 u8 *buf; 128 129 for (; (buf = *bufs); bufs++) { 130 for (i = 0; i < start; i++) 131 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 132 for ( ; i < start + len; i++) 133 buf[i] = PATTERN_DST | PATTERN_OVERWRITE 134 | (~i & PATTERN_COUNT_MASK); 135 for ( ; i < test_buf_size; i++) 136 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 137 } 138 } 139 140 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 141 unsigned int counter, bool is_srcbuf) 142 { 143 u8 diff = actual ^ pattern; 144 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 145 const char *thread_name = current->comm; 146 147 if (is_srcbuf) 148 pr_warning("%s: srcbuf[0x%x] overwritten!" 149 " Expected %02x, got %02x\n", 150 thread_name, index, expected, actual); 151 else if ((pattern & PATTERN_COPY) 152 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 153 pr_warning("%s: dstbuf[0x%x] not copied!" 154 " Expected %02x, got %02x\n", 155 thread_name, index, expected, actual); 156 else if (diff & PATTERN_SRC) 157 pr_warning("%s: dstbuf[0x%x] was copied!" 158 " Expected %02x, got %02x\n", 159 thread_name, index, expected, actual); 160 else 161 pr_warning("%s: dstbuf[0x%x] mismatch!" 162 " Expected %02x, got %02x\n", 163 thread_name, index, expected, actual); 164 } 165 166 static unsigned int dmatest_verify(u8 **bufs, unsigned int start, 167 unsigned int end, unsigned int counter, u8 pattern, 168 bool is_srcbuf) 169 { 170 unsigned int i; 171 unsigned int error_count = 0; 172 u8 actual; 173 u8 expected; 174 u8 *buf; 175 unsigned int counter_orig = counter; 176 177 for (; (buf = *bufs); bufs++) { 178 counter = counter_orig; 179 for (i = start; i < end; i++) { 180 actual = buf[i]; 181 expected = pattern | (~counter & PATTERN_COUNT_MASK); 182 if (actual != expected) { 183 if (error_count < 32) 184 dmatest_mismatch(actual, pattern, i, 185 counter, is_srcbuf); 186 error_count++; 187 } 188 counter++; 189 } 190 } 191 192 if (error_count > 32) 193 pr_warning("%s: %u errors suppressed\n", 194 current->comm, error_count - 32); 195 196 return error_count; 197 } 198 199 static void dmatest_callback(void *completion) 200 { 201 complete(completion); 202 } 203 204 /* 205 * This function repeatedly tests DMA transfers of various lengths and 206 * offsets for a given operation type until it is told to exit by 207 * kthread_stop(). There may be multiple threads running this function 208 * in parallel for a single channel, and there may be multiple channels 209 * being tested in parallel. 210 * 211 * Before each test, the source and destination buffer is initialized 212 * with a known pattern. This pattern is different depending on 213 * whether it's in an area which is supposed to be copied or 214 * overwritten, and different in the source and destination buffers. 215 * So if the DMA engine doesn't copy exactly what we tell it to copy, 216 * we'll notice. 217 */ 218 static int dmatest_func(void *data) 219 { 220 struct dmatest_thread *thread = data; 221 struct dma_chan *chan; 222 const char *thread_name; 223 unsigned int src_off, dst_off, len; 224 unsigned int error_count; 225 unsigned int failed_tests = 0; 226 unsigned int total_tests = 0; 227 dma_cookie_t cookie; 228 enum dma_status status; 229 enum dma_ctrl_flags flags; 230 int ret; 231 int src_cnt; 232 int dst_cnt; 233 int i; 234 235 thread_name = current->comm; 236 237 ret = -ENOMEM; 238 239 smp_rmb(); 240 chan = thread->chan; 241 if (thread->type == DMA_MEMCPY) 242 src_cnt = dst_cnt = 1; 243 else if (thread->type == DMA_XOR) { 244 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */ 245 dst_cnt = 1; 246 } else 247 goto err_srcs; 248 249 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 250 if (!thread->srcs) 251 goto err_srcs; 252 for (i = 0; i < src_cnt; i++) { 253 thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL); 254 if (!thread->srcs[i]) 255 goto err_srcbuf; 256 } 257 thread->srcs[i] = NULL; 258 259 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 260 if (!thread->dsts) 261 goto err_dsts; 262 for (i = 0; i < dst_cnt; i++) { 263 thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL); 264 if (!thread->dsts[i]) 265 goto err_dstbuf; 266 } 267 thread->dsts[i] = NULL; 268 269 set_user_nice(current, 10); 270 271 flags = DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP | DMA_PREP_INTERRUPT; 272 273 while (!kthread_should_stop()) { 274 struct dma_device *dev = chan->device; 275 struct dma_async_tx_descriptor *tx = NULL; 276 dma_addr_t dma_srcs[src_cnt]; 277 dma_addr_t dma_dsts[dst_cnt]; 278 struct completion cmp; 279 unsigned long tmo = msecs_to_jiffies(3000); 280 281 total_tests++; 282 283 len = dmatest_random() % test_buf_size + 1; 284 src_off = dmatest_random() % (test_buf_size - len + 1); 285 dst_off = dmatest_random() % (test_buf_size - len + 1); 286 287 dmatest_init_srcs(thread->srcs, src_off, len); 288 dmatest_init_dsts(thread->dsts, dst_off, len); 289 290 for (i = 0; i < src_cnt; i++) { 291 u8 *buf = thread->srcs[i] + src_off; 292 293 dma_srcs[i] = dma_map_single(dev->dev, buf, len, 294 DMA_TO_DEVICE); 295 } 296 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 297 for (i = 0; i < dst_cnt; i++) { 298 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 299 test_buf_size, 300 DMA_BIDIRECTIONAL); 301 } 302 303 if (thread->type == DMA_MEMCPY) 304 tx = dev->device_prep_dma_memcpy(chan, 305 dma_dsts[0] + dst_off, 306 dma_srcs[0], len, 307 flags); 308 else if (thread->type == DMA_XOR) 309 tx = dev->device_prep_dma_xor(chan, 310 dma_dsts[0] + dst_off, 311 dma_srcs, xor_sources, 312 len, flags); 313 314 if (!tx) { 315 for (i = 0; i < src_cnt; i++) 316 dma_unmap_single(dev->dev, dma_srcs[i], len, 317 DMA_TO_DEVICE); 318 for (i = 0; i < dst_cnt; i++) 319 dma_unmap_single(dev->dev, dma_dsts[i], 320 test_buf_size, 321 DMA_BIDIRECTIONAL); 322 pr_warning("%s: #%u: prep error with src_off=0x%x " 323 "dst_off=0x%x len=0x%x\n", 324 thread_name, total_tests - 1, 325 src_off, dst_off, len); 326 msleep(100); 327 failed_tests++; 328 continue; 329 } 330 331 init_completion(&cmp); 332 tx->callback = dmatest_callback; 333 tx->callback_param = &cmp; 334 cookie = tx->tx_submit(tx); 335 336 if (dma_submit_error(cookie)) { 337 pr_warning("%s: #%u: submit error %d with src_off=0x%x " 338 "dst_off=0x%x len=0x%x\n", 339 thread_name, total_tests - 1, cookie, 340 src_off, dst_off, len); 341 msleep(100); 342 failed_tests++; 343 continue; 344 } 345 dma_async_issue_pending(chan); 346 347 tmo = wait_for_completion_timeout(&cmp, tmo); 348 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 349 350 if (tmo == 0) { 351 pr_warning("%s: #%u: test timed out\n", 352 thread_name, total_tests - 1); 353 failed_tests++; 354 continue; 355 } else if (status != DMA_SUCCESS) { 356 pr_warning("%s: #%u: got completion callback," 357 " but status is \'%s\'\n", 358 thread_name, total_tests - 1, 359 status == DMA_ERROR ? "error" : "in progress"); 360 failed_tests++; 361 continue; 362 } 363 364 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 365 for (i = 0; i < dst_cnt; i++) 366 dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size, 367 DMA_BIDIRECTIONAL); 368 369 error_count = 0; 370 371 pr_debug("%s: verifying source buffer...\n", thread_name); 372 error_count += dmatest_verify(thread->srcs, 0, src_off, 373 0, PATTERN_SRC, true); 374 error_count += dmatest_verify(thread->srcs, src_off, 375 src_off + len, src_off, 376 PATTERN_SRC | PATTERN_COPY, true); 377 error_count += dmatest_verify(thread->srcs, src_off + len, 378 test_buf_size, src_off + len, 379 PATTERN_SRC, true); 380 381 pr_debug("%s: verifying dest buffer...\n", 382 thread->task->comm); 383 error_count += dmatest_verify(thread->dsts, 0, dst_off, 384 0, PATTERN_DST, false); 385 error_count += dmatest_verify(thread->dsts, dst_off, 386 dst_off + len, src_off, 387 PATTERN_SRC | PATTERN_COPY, false); 388 error_count += dmatest_verify(thread->dsts, dst_off + len, 389 test_buf_size, dst_off + len, 390 PATTERN_DST, false); 391 392 if (error_count) { 393 pr_warning("%s: #%u: %u errors with " 394 "src_off=0x%x dst_off=0x%x len=0x%x\n", 395 thread_name, total_tests - 1, error_count, 396 src_off, dst_off, len); 397 failed_tests++; 398 } else { 399 pr_debug("%s: #%u: No errors with " 400 "src_off=0x%x dst_off=0x%x len=0x%x\n", 401 thread_name, total_tests - 1, 402 src_off, dst_off, len); 403 } 404 } 405 406 ret = 0; 407 for (i = 0; thread->dsts[i]; i++) 408 kfree(thread->dsts[i]); 409 err_dstbuf: 410 kfree(thread->dsts); 411 err_dsts: 412 for (i = 0; thread->srcs[i]; i++) 413 kfree(thread->srcs[i]); 414 err_srcbuf: 415 kfree(thread->srcs); 416 err_srcs: 417 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 418 thread_name, total_tests, failed_tests, ret); 419 return ret; 420 } 421 422 static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 423 { 424 struct dmatest_thread *thread; 425 struct dmatest_thread *_thread; 426 int ret; 427 428 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 429 ret = kthread_stop(thread->task); 430 pr_debug("dmatest: thread %s exited with status %d\n", 431 thread->task->comm, ret); 432 list_del(&thread->node); 433 kfree(thread); 434 } 435 kfree(dtc); 436 } 437 438 static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type) 439 { 440 struct dmatest_thread *thread; 441 struct dma_chan *chan = dtc->chan; 442 char *op; 443 unsigned int i; 444 445 if (type == DMA_MEMCPY) 446 op = "copy"; 447 else if (type == DMA_XOR) 448 op = "xor"; 449 else 450 return -EINVAL; 451 452 for (i = 0; i < threads_per_chan; i++) { 453 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 454 if (!thread) { 455 pr_warning("dmatest: No memory for %s-%s%u\n", 456 dma_chan_name(chan), op, i); 457 458 break; 459 } 460 thread->chan = dtc->chan; 461 thread->type = type; 462 smp_wmb(); 463 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 464 dma_chan_name(chan), op, i); 465 if (IS_ERR(thread->task)) { 466 pr_warning("dmatest: Failed to run thread %s-%s%u\n", 467 dma_chan_name(chan), op, i); 468 kfree(thread); 469 break; 470 } 471 472 /* srcbuf and dstbuf are allocated by the thread itself */ 473 474 list_add_tail(&thread->node, &dtc->threads); 475 } 476 477 return i; 478 } 479 480 static int dmatest_add_channel(struct dma_chan *chan) 481 { 482 struct dmatest_chan *dtc; 483 struct dma_device *dma_dev = chan->device; 484 unsigned int thread_count = 0; 485 unsigned int cnt; 486 487 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 488 if (!dtc) { 489 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 490 return -ENOMEM; 491 } 492 493 dtc->chan = chan; 494 INIT_LIST_HEAD(&dtc->threads); 495 496 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 497 cnt = dmatest_add_threads(dtc, DMA_MEMCPY); 498 thread_count += cnt > 0 ?: 0; 499 } 500 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 501 cnt = dmatest_add_threads(dtc, DMA_XOR); 502 thread_count += cnt > 0 ?: 0; 503 } 504 505 pr_info("dmatest: Started %u threads using %s\n", 506 thread_count, dma_chan_name(chan)); 507 508 list_add_tail(&dtc->node, &dmatest_channels); 509 nr_channels++; 510 511 return 0; 512 } 513 514 static bool filter(struct dma_chan *chan, void *param) 515 { 516 if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device)) 517 return false; 518 else 519 return true; 520 } 521 522 static int __init dmatest_init(void) 523 { 524 dma_cap_mask_t mask; 525 struct dma_chan *chan; 526 int err = 0; 527 528 dma_cap_zero(mask); 529 dma_cap_set(DMA_MEMCPY, mask); 530 for (;;) { 531 chan = dma_request_channel(mask, filter, NULL); 532 if (chan) { 533 err = dmatest_add_channel(chan); 534 if (err == 0) 535 continue; 536 else { 537 dma_release_channel(chan); 538 break; /* add_channel failed, punt */ 539 } 540 } else 541 break; /* no more channels available */ 542 if (max_channels && nr_channels >= max_channels) 543 break; /* we have all we need */ 544 } 545 546 return err; 547 } 548 /* when compiled-in wait for drivers to load first */ 549 late_initcall(dmatest_init); 550 551 static void __exit dmatest_exit(void) 552 { 553 struct dmatest_chan *dtc, *_dtc; 554 struct dma_chan *chan; 555 556 list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) { 557 list_del(&dtc->node); 558 chan = dtc->chan; 559 dmatest_cleanup_channel(dtc); 560 pr_debug("dmatest: dropped channel %s\n", 561 dma_chan_name(chan)); 562 dma_release_channel(chan); 563 } 564 } 565 module_exit(dmatest_exit); 566 567 MODULE_AUTHOR("Haavard Skinnemoen <hskinnemoen@atmel.com>"); 568 MODULE_LICENSE("GPL v2"); 569