1 /* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/delay.h> 11 #include <linux/dmaengine.h> 12 #include <linux/init.h> 13 #include <linux/kthread.h> 14 #include <linux/module.h> 15 #include <linux/moduleparam.h> 16 #include <linux/random.h> 17 #include <linux/wait.h> 18 19 static unsigned int test_buf_size = 16384; 20 module_param(test_buf_size, uint, S_IRUGO); 21 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 22 23 static char test_channel[20]; 24 module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO); 25 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 26 27 static char test_device[20]; 28 module_param_string(device, test_device, sizeof(test_device), S_IRUGO); 29 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 30 31 static unsigned int threads_per_chan = 1; 32 module_param(threads_per_chan, uint, S_IRUGO); 33 MODULE_PARM_DESC(threads_per_chan, 34 "Number of threads to start per channel (default: 1)"); 35 36 static unsigned int max_channels; 37 module_param(max_channels, uint, S_IRUGO); 38 MODULE_PARM_DESC(max_channels, 39 "Maximum number of channels to use (default: all)"); 40 41 /* 42 * Initialization patterns. All bytes in the source buffer has bit 7 43 * set, all bytes in the destination buffer has bit 7 cleared. 44 * 45 * Bit 6 is set for all bytes which are to be copied by the DMA 46 * engine. Bit 5 is set for all bytes which are to be overwritten by 47 * the DMA engine. 48 * 49 * The remaining bits are the inverse of a counter which increments by 50 * one for each byte address. 51 */ 52 #define PATTERN_SRC 0x80 53 #define PATTERN_DST 0x00 54 #define PATTERN_COPY 0x40 55 #define PATTERN_OVERWRITE 0x20 56 #define PATTERN_COUNT_MASK 0x1f 57 58 struct dmatest_thread { 59 struct list_head node; 60 struct task_struct *task; 61 struct dma_chan *chan; 62 u8 *srcbuf; 63 u8 *dstbuf; 64 }; 65 66 struct dmatest_chan { 67 struct list_head node; 68 struct dma_chan *chan; 69 struct list_head threads; 70 }; 71 72 /* 73 * These are protected by dma_list_mutex since they're only used by 74 * the DMA filter function callback 75 */ 76 static LIST_HEAD(dmatest_channels); 77 static unsigned int nr_channels; 78 79 static bool dmatest_match_channel(struct dma_chan *chan) 80 { 81 if (test_channel[0] == '\0') 82 return true; 83 return strcmp(dma_chan_name(chan), test_channel) == 0; 84 } 85 86 static bool dmatest_match_device(struct dma_device *device) 87 { 88 if (test_device[0] == '\0') 89 return true; 90 return strcmp(dev_name(device->dev), test_device) == 0; 91 } 92 93 static unsigned long dmatest_random(void) 94 { 95 unsigned long buf; 96 97 get_random_bytes(&buf, sizeof(buf)); 98 return buf; 99 } 100 101 static void dmatest_init_srcbuf(u8 *buf, unsigned int start, unsigned int len) 102 { 103 unsigned int i; 104 105 for (i = 0; i < start; i++) 106 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 107 for ( ; i < start + len; i++) 108 buf[i] = PATTERN_SRC | PATTERN_COPY 109 | (~i & PATTERN_COUNT_MASK);; 110 for ( ; i < test_buf_size; i++) 111 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 112 } 113 114 static void dmatest_init_dstbuf(u8 *buf, unsigned int start, unsigned int len) 115 { 116 unsigned int i; 117 118 for (i = 0; i < start; i++) 119 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 120 for ( ; i < start + len; i++) 121 buf[i] = PATTERN_DST | PATTERN_OVERWRITE 122 | (~i & PATTERN_COUNT_MASK); 123 for ( ; i < test_buf_size; i++) 124 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 125 } 126 127 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 128 unsigned int counter, bool is_srcbuf) 129 { 130 u8 diff = actual ^ pattern; 131 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 132 const char *thread_name = current->comm; 133 134 if (is_srcbuf) 135 pr_warning("%s: srcbuf[0x%x] overwritten!" 136 " Expected %02x, got %02x\n", 137 thread_name, index, expected, actual); 138 else if ((pattern & PATTERN_COPY) 139 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 140 pr_warning("%s: dstbuf[0x%x] not copied!" 141 " Expected %02x, got %02x\n", 142 thread_name, index, expected, actual); 143 else if (diff & PATTERN_SRC) 144 pr_warning("%s: dstbuf[0x%x] was copied!" 145 " Expected %02x, got %02x\n", 146 thread_name, index, expected, actual); 147 else 148 pr_warning("%s: dstbuf[0x%x] mismatch!" 149 " Expected %02x, got %02x\n", 150 thread_name, index, expected, actual); 151 } 152 153 static unsigned int dmatest_verify(u8 *buf, unsigned int start, 154 unsigned int end, unsigned int counter, u8 pattern, 155 bool is_srcbuf) 156 { 157 unsigned int i; 158 unsigned int error_count = 0; 159 u8 actual; 160 161 for (i = start; i < end; i++) { 162 actual = buf[i]; 163 if (actual != (pattern | (~counter & PATTERN_COUNT_MASK))) { 164 if (error_count < 32) 165 dmatest_mismatch(actual, pattern, i, counter, 166 is_srcbuf); 167 error_count++; 168 } 169 counter++; 170 } 171 172 if (error_count > 32) 173 pr_warning("%s: %u errors suppressed\n", 174 current->comm, error_count - 32); 175 176 return error_count; 177 } 178 179 /* 180 * This function repeatedly tests DMA transfers of various lengths and 181 * offsets until it is told to exit by kthread_stop(). There may be 182 * multiple threads running this function in parallel for a single 183 * channel, and there may be multiple channels being tested in 184 * parallel. 185 * 186 * Before each test, the source and destination buffer is initialized 187 * with a known pattern. This pattern is different depending on 188 * whether it's in an area which is supposed to be copied or 189 * overwritten, and different in the source and destination buffers. 190 * So if the DMA engine doesn't copy exactly what we tell it to copy, 191 * we'll notice. 192 */ 193 static int dmatest_func(void *data) 194 { 195 struct dmatest_thread *thread = data; 196 struct dma_chan *chan; 197 const char *thread_name; 198 unsigned int src_off, dst_off, len; 199 unsigned int error_count; 200 unsigned int failed_tests = 0; 201 unsigned int total_tests = 0; 202 dma_cookie_t cookie; 203 enum dma_status status; 204 int ret; 205 206 thread_name = current->comm; 207 208 ret = -ENOMEM; 209 thread->srcbuf = kmalloc(test_buf_size, GFP_KERNEL); 210 if (!thread->srcbuf) 211 goto err_srcbuf; 212 thread->dstbuf = kmalloc(test_buf_size, GFP_KERNEL); 213 if (!thread->dstbuf) 214 goto err_dstbuf; 215 216 smp_rmb(); 217 chan = thread->chan; 218 219 while (!kthread_should_stop()) { 220 struct dma_device *dev = chan->device; 221 struct dma_async_tx_descriptor *tx; 222 dma_addr_t dma_src, dma_dest; 223 224 total_tests++; 225 226 len = dmatest_random() % test_buf_size + 1; 227 src_off = dmatest_random() % (test_buf_size - len + 1); 228 dst_off = dmatest_random() % (test_buf_size - len + 1); 229 230 dmatest_init_srcbuf(thread->srcbuf, src_off, len); 231 dmatest_init_dstbuf(thread->dstbuf, dst_off, len); 232 233 dma_src = dma_map_single(dev->dev, thread->srcbuf + src_off, 234 len, DMA_TO_DEVICE); 235 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 236 dma_dest = dma_map_single(dev->dev, thread->dstbuf, 237 test_buf_size, DMA_BIDIRECTIONAL); 238 239 tx = dev->device_prep_dma_memcpy(chan, dma_dest + dst_off, 240 dma_src, len, 241 DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP); 242 if (!tx) { 243 dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 244 dma_unmap_single(dev->dev, dma_dest, 245 test_buf_size, DMA_BIDIRECTIONAL); 246 pr_warning("%s: #%u: prep error with src_off=0x%x " 247 "dst_off=0x%x len=0x%x\n", 248 thread_name, total_tests - 1, 249 src_off, dst_off, len); 250 msleep(100); 251 failed_tests++; 252 continue; 253 } 254 tx->callback = NULL; 255 cookie = tx->tx_submit(tx); 256 257 if (dma_submit_error(cookie)) { 258 pr_warning("%s: #%u: submit error %d with src_off=0x%x " 259 "dst_off=0x%x len=0x%x\n", 260 thread_name, total_tests - 1, cookie, 261 src_off, dst_off, len); 262 msleep(100); 263 failed_tests++; 264 continue; 265 } 266 dma_async_memcpy_issue_pending(chan); 267 268 do { 269 msleep(1); 270 status = dma_async_memcpy_complete( 271 chan, cookie, NULL, NULL); 272 } while (status == DMA_IN_PROGRESS); 273 274 if (status == DMA_ERROR) { 275 pr_warning("%s: #%u: error during copy\n", 276 thread_name, total_tests - 1); 277 failed_tests++; 278 continue; 279 } 280 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 281 dma_unmap_single(dev->dev, dma_dest, 282 test_buf_size, DMA_BIDIRECTIONAL); 283 284 error_count = 0; 285 286 pr_debug("%s: verifying source buffer...\n", thread_name); 287 error_count += dmatest_verify(thread->srcbuf, 0, src_off, 288 0, PATTERN_SRC, true); 289 error_count += dmatest_verify(thread->srcbuf, src_off, 290 src_off + len, src_off, 291 PATTERN_SRC | PATTERN_COPY, true); 292 error_count += dmatest_verify(thread->srcbuf, src_off + len, 293 test_buf_size, src_off + len, 294 PATTERN_SRC, true); 295 296 pr_debug("%s: verifying dest buffer...\n", 297 thread->task->comm); 298 error_count += dmatest_verify(thread->dstbuf, 0, dst_off, 299 0, PATTERN_DST, false); 300 error_count += dmatest_verify(thread->dstbuf, dst_off, 301 dst_off + len, src_off, 302 PATTERN_SRC | PATTERN_COPY, false); 303 error_count += dmatest_verify(thread->dstbuf, dst_off + len, 304 test_buf_size, dst_off + len, 305 PATTERN_DST, false); 306 307 if (error_count) { 308 pr_warning("%s: #%u: %u errors with " 309 "src_off=0x%x dst_off=0x%x len=0x%x\n", 310 thread_name, total_tests - 1, error_count, 311 src_off, dst_off, len); 312 failed_tests++; 313 } else { 314 pr_debug("%s: #%u: No errors with " 315 "src_off=0x%x dst_off=0x%x len=0x%x\n", 316 thread_name, total_tests - 1, 317 src_off, dst_off, len); 318 } 319 } 320 321 ret = 0; 322 kfree(thread->dstbuf); 323 err_dstbuf: 324 kfree(thread->srcbuf); 325 err_srcbuf: 326 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 327 thread_name, total_tests, failed_tests, ret); 328 return ret; 329 } 330 331 static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 332 { 333 struct dmatest_thread *thread; 334 struct dmatest_thread *_thread; 335 int ret; 336 337 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 338 ret = kthread_stop(thread->task); 339 pr_debug("dmatest: thread %s exited with status %d\n", 340 thread->task->comm, ret); 341 list_del(&thread->node); 342 kfree(thread); 343 } 344 kfree(dtc); 345 } 346 347 static int dmatest_add_channel(struct dma_chan *chan) 348 { 349 struct dmatest_chan *dtc; 350 struct dmatest_thread *thread; 351 unsigned int i; 352 353 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 354 if (!dtc) { 355 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 356 return -ENOMEM; 357 } 358 359 dtc->chan = chan; 360 INIT_LIST_HEAD(&dtc->threads); 361 362 for (i = 0; i < threads_per_chan; i++) { 363 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 364 if (!thread) { 365 pr_warning("dmatest: No memory for %s-test%u\n", 366 dma_chan_name(chan), i); 367 break; 368 } 369 thread->chan = dtc->chan; 370 smp_wmb(); 371 thread->task = kthread_run(dmatest_func, thread, "%s-test%u", 372 dma_chan_name(chan), i); 373 if (IS_ERR(thread->task)) { 374 pr_warning("dmatest: Failed to run thread %s-test%u\n", 375 dma_chan_name(chan), i); 376 kfree(thread); 377 break; 378 } 379 380 /* srcbuf and dstbuf are allocated by the thread itself */ 381 382 list_add_tail(&thread->node, &dtc->threads); 383 } 384 385 pr_info("dmatest: Started %u threads using %s\n", i, dma_chan_name(chan)); 386 387 list_add_tail(&dtc->node, &dmatest_channels); 388 nr_channels++; 389 390 return 0; 391 } 392 393 static bool filter(struct dma_chan *chan, void *param) 394 { 395 if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device)) 396 return false; 397 else 398 return true; 399 } 400 401 static int __init dmatest_init(void) 402 { 403 dma_cap_mask_t mask; 404 struct dma_chan *chan; 405 int err = 0; 406 407 dma_cap_zero(mask); 408 dma_cap_set(DMA_MEMCPY, mask); 409 for (;;) { 410 chan = dma_request_channel(mask, filter, NULL); 411 if (chan) { 412 err = dmatest_add_channel(chan); 413 if (err == 0) 414 continue; 415 else { 416 dma_release_channel(chan); 417 break; /* add_channel failed, punt */ 418 } 419 } else 420 break; /* no more channels available */ 421 if (max_channels && nr_channels >= max_channels) 422 break; /* we have all we need */ 423 } 424 425 return err; 426 } 427 /* when compiled-in wait for drivers to load first */ 428 late_initcall(dmatest_init); 429 430 static void __exit dmatest_exit(void) 431 { 432 struct dmatest_chan *dtc, *_dtc; 433 struct dma_chan *chan; 434 435 list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) { 436 list_del(&dtc->node); 437 chan = dtc->chan; 438 dmatest_cleanup_channel(dtc); 439 pr_debug("dmatest: dropped channel %s\n", 440 dma_chan_name(chan)); 441 dma_release_channel(chan); 442 } 443 } 444 module_exit(dmatest_exit); 445 446 MODULE_AUTHOR("Haavard Skinnemoen <hskinnemoen@atmel.com>"); 447 MODULE_LICENSE("GPL v2"); 448