1 /* 2 * AMD Cryptographic Coprocessor (CCP) driver 3 * 4 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. 5 * 6 * Author: Gary R Hook <gary.hook@amd.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/kernel.h> 15 #include <linux/dmaengine.h> 16 #include <linux/spinlock.h> 17 #include <linux/mutex.h> 18 #include <linux/ccp.h> 19 20 #include "ccp-dev.h" 21 #include "../../dma/dmaengine.h" 22 23 #define CCP_DMA_WIDTH(_mask) \ 24 ({ \ 25 u64 mask = _mask + 1; \ 26 (mask == 0) ? 64 : fls64(mask); \ 27 }) 28 29 /* The CCP as a DMA provider can be configured for public or private 30 * channels. Default is specified in the vdata for the device (PCI ID). 31 * This module parameter will override for all channels on all devices: 32 * dma_chan_attr = 0x2 to force all channels public 33 * = 0x1 to force all channels private 34 * = 0x0 to defer to the vdata setting 35 * = any other value: warning, revert to 0x0 36 */ 37 static unsigned int dma_chan_attr = CCP_DMA_DFLT; 38 module_param(dma_chan_attr, uint, 0444); 39 MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); 40 41 static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) 42 { 43 switch (dma_chan_attr) { 44 case CCP_DMA_DFLT: 45 return ccp->vdata->dma_chan_attr; 46 47 case CCP_DMA_PRIV: 48 return DMA_PRIVATE; 49 50 case CCP_DMA_PUB: 51 return 0; 52 53 default: 54 dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", 55 dma_chan_attr); 56 return ccp->vdata->dma_chan_attr; 57 } 58 } 59 60 static void ccp_free_cmd_resources(struct ccp_device *ccp, 61 struct list_head *list) 62 { 63 struct ccp_dma_cmd *cmd, *ctmp; 64 65 list_for_each_entry_safe(cmd, ctmp, list, entry) { 66 list_del(&cmd->entry); 67 kmem_cache_free(ccp->dma_cmd_cache, cmd); 68 } 69 } 70 71 static void ccp_free_desc_resources(struct ccp_device *ccp, 72 struct list_head *list) 73 { 74 struct ccp_dma_desc *desc, *dtmp; 75 76 list_for_each_entry_safe(desc, dtmp, list, entry) { 77 ccp_free_cmd_resources(ccp, &desc->active); 78 ccp_free_cmd_resources(ccp, &desc->pending); 79 80 list_del(&desc->entry); 81 kmem_cache_free(ccp->dma_desc_cache, desc); 82 } 83 } 84 85 static void ccp_free_chan_resources(struct dma_chan *dma_chan) 86 { 87 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 88 dma_chan); 89 unsigned long flags; 90 91 dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); 92 93 spin_lock_irqsave(&chan->lock, flags); 94 95 ccp_free_desc_resources(chan->ccp, &chan->complete); 96 ccp_free_desc_resources(chan->ccp, &chan->active); 97 ccp_free_desc_resources(chan->ccp, &chan->pending); 98 ccp_free_desc_resources(chan->ccp, &chan->created); 99 100 spin_unlock_irqrestore(&chan->lock, flags); 101 } 102 103 static void ccp_cleanup_desc_resources(struct ccp_device *ccp, 104 struct list_head *list) 105 { 106 struct ccp_dma_desc *desc, *dtmp; 107 108 list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { 109 if (!async_tx_test_ack(&desc->tx_desc)) 110 continue; 111 112 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 113 114 ccp_free_cmd_resources(ccp, &desc->active); 115 ccp_free_cmd_resources(ccp, &desc->pending); 116 117 list_del(&desc->entry); 118 kmem_cache_free(ccp->dma_desc_cache, desc); 119 } 120 } 121 122 static void ccp_do_cleanup(unsigned long data) 123 { 124 struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; 125 unsigned long flags; 126 127 dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, 128 dma_chan_name(&chan->dma_chan)); 129 130 spin_lock_irqsave(&chan->lock, flags); 131 132 ccp_cleanup_desc_resources(chan->ccp, &chan->complete); 133 134 spin_unlock_irqrestore(&chan->lock, flags); 135 } 136 137 static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) 138 { 139 struct ccp_dma_cmd *cmd; 140 int ret; 141 142 cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); 143 list_move(&cmd->entry, &desc->active); 144 145 dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, 146 desc->tx_desc.cookie, cmd); 147 148 ret = ccp_enqueue_cmd(&cmd->ccp_cmd); 149 if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) 150 return 0; 151 152 dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, 153 ret, desc->tx_desc.cookie, cmd); 154 155 return ret; 156 } 157 158 static void ccp_free_active_cmd(struct ccp_dma_desc *desc) 159 { 160 struct ccp_dma_cmd *cmd; 161 162 cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, 163 entry); 164 if (!cmd) 165 return; 166 167 dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", 168 __func__, desc->tx_desc.cookie, cmd); 169 170 list_del(&cmd->entry); 171 kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); 172 } 173 174 static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, 175 struct ccp_dma_desc *desc) 176 { 177 /* Move current DMA descriptor to the complete list */ 178 if (desc) 179 list_move(&desc->entry, &chan->complete); 180 181 /* Get the next DMA descriptor on the active list */ 182 desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 183 entry); 184 185 return desc; 186 } 187 188 static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, 189 struct ccp_dma_desc *desc) 190 { 191 struct dma_async_tx_descriptor *tx_desc; 192 unsigned long flags; 193 194 /* Loop over descriptors until one is found with commands */ 195 do { 196 if (desc) { 197 /* Remove the DMA command from the list and free it */ 198 ccp_free_active_cmd(desc); 199 200 if (!list_empty(&desc->pending)) { 201 /* No errors, keep going */ 202 if (desc->status != DMA_ERROR) 203 return desc; 204 205 /* Error, free remaining commands and move on */ 206 ccp_free_cmd_resources(desc->ccp, 207 &desc->pending); 208 } 209 210 tx_desc = &desc->tx_desc; 211 } else { 212 tx_desc = NULL; 213 } 214 215 spin_lock_irqsave(&chan->lock, flags); 216 217 if (desc) { 218 if (desc->status != DMA_ERROR) 219 desc->status = DMA_COMPLETE; 220 221 dev_dbg(desc->ccp->dev, 222 "%s - tx %d complete, status=%u\n", __func__, 223 desc->tx_desc.cookie, desc->status); 224 225 dma_cookie_complete(tx_desc); 226 dma_descriptor_unmap(tx_desc); 227 } 228 229 desc = __ccp_next_dma_desc(chan, desc); 230 231 spin_unlock_irqrestore(&chan->lock, flags); 232 233 if (tx_desc) { 234 dmaengine_desc_get_callback_invoke(tx_desc, NULL); 235 236 dma_run_dependencies(tx_desc); 237 } 238 } while (desc); 239 240 return NULL; 241 } 242 243 static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) 244 { 245 struct ccp_dma_desc *desc; 246 247 if (list_empty(&chan->pending)) 248 return NULL; 249 250 desc = list_empty(&chan->active) 251 ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) 252 : NULL; 253 254 list_splice_tail_init(&chan->pending, &chan->active); 255 256 return desc; 257 } 258 259 static void ccp_cmd_callback(void *data, int err) 260 { 261 struct ccp_dma_desc *desc = data; 262 struct ccp_dma_chan *chan; 263 int ret; 264 265 if (err == -EINPROGRESS) 266 return; 267 268 chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, 269 dma_chan); 270 271 dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", 272 __func__, desc->tx_desc.cookie, err); 273 274 if (err) 275 desc->status = DMA_ERROR; 276 277 while (true) { 278 /* Check for DMA descriptor completion */ 279 desc = ccp_handle_active_desc(chan, desc); 280 281 /* Don't submit cmd if no descriptor or DMA is paused */ 282 if (!desc || (chan->status == DMA_PAUSED)) 283 break; 284 285 ret = ccp_issue_next_cmd(desc); 286 if (!ret) 287 break; 288 289 desc->status = DMA_ERROR; 290 } 291 292 tasklet_schedule(&chan->cleanup_tasklet); 293 } 294 295 static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) 296 { 297 struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, 298 tx_desc); 299 struct ccp_dma_chan *chan; 300 dma_cookie_t cookie; 301 unsigned long flags; 302 303 chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); 304 305 spin_lock_irqsave(&chan->lock, flags); 306 307 cookie = dma_cookie_assign(tx_desc); 308 list_del(&desc->entry); 309 list_add_tail(&desc->entry, &chan->pending); 310 311 spin_unlock_irqrestore(&chan->lock, flags); 312 313 dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", 314 __func__, cookie); 315 316 return cookie; 317 } 318 319 static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) 320 { 321 struct ccp_dma_cmd *cmd; 322 323 cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); 324 if (cmd) 325 memset(cmd, 0, sizeof(*cmd)); 326 327 return cmd; 328 } 329 330 static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, 331 unsigned long flags) 332 { 333 struct ccp_dma_desc *desc; 334 335 desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); 336 if (!desc) 337 return NULL; 338 339 dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); 340 desc->tx_desc.flags = flags; 341 desc->tx_desc.tx_submit = ccp_tx_submit; 342 desc->ccp = chan->ccp; 343 INIT_LIST_HEAD(&desc->pending); 344 INIT_LIST_HEAD(&desc->active); 345 desc->status = DMA_IN_PROGRESS; 346 347 return desc; 348 } 349 350 static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, 351 struct scatterlist *dst_sg, 352 unsigned int dst_nents, 353 struct scatterlist *src_sg, 354 unsigned int src_nents, 355 unsigned long flags) 356 { 357 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 358 dma_chan); 359 struct ccp_device *ccp = chan->ccp; 360 struct ccp_dma_desc *desc; 361 struct ccp_dma_cmd *cmd; 362 struct ccp_cmd *ccp_cmd; 363 struct ccp_passthru_nomap_engine *ccp_pt; 364 unsigned int src_offset, src_len; 365 unsigned int dst_offset, dst_len; 366 unsigned int len; 367 unsigned long sflags; 368 size_t total_len; 369 370 if (!dst_sg || !src_sg) 371 return NULL; 372 373 if (!dst_nents || !src_nents) 374 return NULL; 375 376 desc = ccp_alloc_dma_desc(chan, flags); 377 if (!desc) 378 return NULL; 379 380 total_len = 0; 381 382 src_len = sg_dma_len(src_sg); 383 src_offset = 0; 384 385 dst_len = sg_dma_len(dst_sg); 386 dst_offset = 0; 387 388 while (true) { 389 if (!src_len) { 390 src_nents--; 391 if (!src_nents) 392 break; 393 394 src_sg = sg_next(src_sg); 395 if (!src_sg) 396 break; 397 398 src_len = sg_dma_len(src_sg); 399 src_offset = 0; 400 continue; 401 } 402 403 if (!dst_len) { 404 dst_nents--; 405 if (!dst_nents) 406 break; 407 408 dst_sg = sg_next(dst_sg); 409 if (!dst_sg) 410 break; 411 412 dst_len = sg_dma_len(dst_sg); 413 dst_offset = 0; 414 continue; 415 } 416 417 len = min(dst_len, src_len); 418 419 cmd = ccp_alloc_dma_cmd(chan); 420 if (!cmd) 421 goto err; 422 423 ccp_cmd = &cmd->ccp_cmd; 424 ccp_cmd->ccp = chan->ccp; 425 ccp_pt = &ccp_cmd->u.passthru_nomap; 426 ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; 427 ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; 428 ccp_cmd->engine = CCP_ENGINE_PASSTHRU; 429 ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; 430 ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; 431 ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; 432 ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; 433 ccp_pt->src_len = len; 434 ccp_pt->final = 1; 435 ccp_cmd->callback = ccp_cmd_callback; 436 ccp_cmd->data = desc; 437 438 list_add_tail(&cmd->entry, &desc->pending); 439 440 dev_dbg(ccp->dev, 441 "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, 442 cmd, &ccp_pt->src_dma, 443 &ccp_pt->dst_dma, ccp_pt->src_len); 444 445 total_len += len; 446 447 src_len -= len; 448 src_offset += len; 449 450 dst_len -= len; 451 dst_offset += len; 452 } 453 454 desc->len = total_len; 455 456 if (list_empty(&desc->pending)) 457 goto err; 458 459 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 460 461 spin_lock_irqsave(&chan->lock, sflags); 462 463 list_add_tail(&desc->entry, &chan->created); 464 465 spin_unlock_irqrestore(&chan->lock, sflags); 466 467 return desc; 468 469 err: 470 ccp_free_cmd_resources(ccp, &desc->pending); 471 kmem_cache_free(ccp->dma_desc_cache, desc); 472 473 return NULL; 474 } 475 476 static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( 477 struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, 478 unsigned long flags) 479 { 480 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 481 dma_chan); 482 struct ccp_dma_desc *desc; 483 struct scatterlist dst_sg, src_sg; 484 485 dev_dbg(chan->ccp->dev, 486 "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", 487 __func__, &src, &dst, len, flags); 488 489 sg_init_table(&dst_sg, 1); 490 sg_dma_address(&dst_sg) = dst; 491 sg_dma_len(&dst_sg) = len; 492 493 sg_init_table(&src_sg, 1); 494 sg_dma_address(&src_sg) = src; 495 sg_dma_len(&src_sg) = len; 496 497 desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); 498 if (!desc) 499 return NULL; 500 501 return &desc->tx_desc; 502 } 503 504 static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( 505 struct dma_chan *dma_chan, unsigned long flags) 506 { 507 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 508 dma_chan); 509 struct ccp_dma_desc *desc; 510 511 desc = ccp_alloc_dma_desc(chan, flags); 512 if (!desc) 513 return NULL; 514 515 return &desc->tx_desc; 516 } 517 518 static void ccp_issue_pending(struct dma_chan *dma_chan) 519 { 520 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 521 dma_chan); 522 struct ccp_dma_desc *desc; 523 unsigned long flags; 524 525 dev_dbg(chan->ccp->dev, "%s\n", __func__); 526 527 spin_lock_irqsave(&chan->lock, flags); 528 529 desc = __ccp_pending_to_active(chan); 530 531 spin_unlock_irqrestore(&chan->lock, flags); 532 533 /* If there was nothing active, start processing */ 534 if (desc) 535 ccp_cmd_callback(desc, 0); 536 } 537 538 static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, 539 dma_cookie_t cookie, 540 struct dma_tx_state *state) 541 { 542 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 543 dma_chan); 544 struct ccp_dma_desc *desc; 545 enum dma_status ret; 546 unsigned long flags; 547 548 if (chan->status == DMA_PAUSED) { 549 ret = DMA_PAUSED; 550 goto out; 551 } 552 553 ret = dma_cookie_status(dma_chan, cookie, state); 554 if (ret == DMA_COMPLETE) { 555 spin_lock_irqsave(&chan->lock, flags); 556 557 /* Get status from complete chain, if still there */ 558 list_for_each_entry(desc, &chan->complete, entry) { 559 if (desc->tx_desc.cookie != cookie) 560 continue; 561 562 ret = desc->status; 563 break; 564 } 565 566 spin_unlock_irqrestore(&chan->lock, flags); 567 } 568 569 out: 570 dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); 571 572 return ret; 573 } 574 575 static int ccp_pause(struct dma_chan *dma_chan) 576 { 577 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 578 dma_chan); 579 580 chan->status = DMA_PAUSED; 581 582 /*TODO: Wait for active DMA to complete before returning? */ 583 584 return 0; 585 } 586 587 static int ccp_resume(struct dma_chan *dma_chan) 588 { 589 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 590 dma_chan); 591 struct ccp_dma_desc *desc; 592 unsigned long flags; 593 594 spin_lock_irqsave(&chan->lock, flags); 595 596 desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 597 entry); 598 599 spin_unlock_irqrestore(&chan->lock, flags); 600 601 /* Indicate the channel is running again */ 602 chan->status = DMA_IN_PROGRESS; 603 604 /* If there was something active, re-start */ 605 if (desc) 606 ccp_cmd_callback(desc, 0); 607 608 return 0; 609 } 610 611 static int ccp_terminate_all(struct dma_chan *dma_chan) 612 { 613 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 614 dma_chan); 615 unsigned long flags; 616 617 dev_dbg(chan->ccp->dev, "%s\n", __func__); 618 619 /*TODO: Wait for active DMA to complete before continuing */ 620 621 spin_lock_irqsave(&chan->lock, flags); 622 623 /*TODO: Purge the complete list? */ 624 ccp_free_desc_resources(chan->ccp, &chan->active); 625 ccp_free_desc_resources(chan->ccp, &chan->pending); 626 ccp_free_desc_resources(chan->ccp, &chan->created); 627 628 spin_unlock_irqrestore(&chan->lock, flags); 629 630 return 0; 631 } 632 633 int ccp_dmaengine_register(struct ccp_device *ccp) 634 { 635 struct ccp_dma_chan *chan; 636 struct dma_device *dma_dev = &ccp->dma_dev; 637 struct dma_chan *dma_chan; 638 char *dma_cmd_cache_name; 639 char *dma_desc_cache_name; 640 unsigned int i; 641 int ret; 642 643 ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, 644 sizeof(*(ccp->ccp_dma_chan)), 645 GFP_KERNEL); 646 if (!ccp->ccp_dma_chan) 647 return -ENOMEM; 648 649 dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 650 "%s-dmaengine-cmd-cache", 651 ccp->name); 652 if (!dma_cmd_cache_name) 653 return -ENOMEM; 654 655 ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, 656 sizeof(struct ccp_dma_cmd), 657 sizeof(void *), 658 SLAB_HWCACHE_ALIGN, NULL); 659 if (!ccp->dma_cmd_cache) 660 return -ENOMEM; 661 662 dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 663 "%s-dmaengine-desc-cache", 664 ccp->name); 665 if (!dma_desc_cache_name) { 666 ret = -ENOMEM; 667 goto err_cache; 668 } 669 670 ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, 671 sizeof(struct ccp_dma_desc), 672 sizeof(void *), 673 SLAB_HWCACHE_ALIGN, NULL); 674 if (!ccp->dma_desc_cache) { 675 ret = -ENOMEM; 676 goto err_cache; 677 } 678 679 dma_dev->dev = ccp->dev; 680 dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 681 dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 682 dma_dev->directions = DMA_MEM_TO_MEM; 683 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 684 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 685 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 686 687 /* The DMA channels for this device can be set to public or private, 688 * and overridden by the module parameter dma_chan_attr. 689 * Default: according to the value in vdata (dma_chan_attr=0) 690 * dma_chan_attr=0x1: all channels private (override vdata) 691 * dma_chan_attr=0x2: all channels public (override vdata) 692 */ 693 if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) 694 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); 695 696 INIT_LIST_HEAD(&dma_dev->channels); 697 for (i = 0; i < ccp->cmd_q_count; i++) { 698 chan = ccp->ccp_dma_chan + i; 699 dma_chan = &chan->dma_chan; 700 701 chan->ccp = ccp; 702 703 spin_lock_init(&chan->lock); 704 INIT_LIST_HEAD(&chan->created); 705 INIT_LIST_HEAD(&chan->pending); 706 INIT_LIST_HEAD(&chan->active); 707 INIT_LIST_HEAD(&chan->complete); 708 709 tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, 710 (unsigned long)chan); 711 712 dma_chan->device = dma_dev; 713 dma_cookie_init(dma_chan); 714 715 list_add_tail(&dma_chan->device_node, &dma_dev->channels); 716 } 717 718 dma_dev->device_free_chan_resources = ccp_free_chan_resources; 719 dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; 720 dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; 721 dma_dev->device_issue_pending = ccp_issue_pending; 722 dma_dev->device_tx_status = ccp_tx_status; 723 dma_dev->device_pause = ccp_pause; 724 dma_dev->device_resume = ccp_resume; 725 dma_dev->device_terminate_all = ccp_terminate_all; 726 727 ret = dma_async_device_register(dma_dev); 728 if (ret) 729 goto err_reg; 730 731 return 0; 732 733 err_reg: 734 kmem_cache_destroy(ccp->dma_desc_cache); 735 736 err_cache: 737 kmem_cache_destroy(ccp->dma_cmd_cache); 738 739 return ret; 740 } 741 742 void ccp_dmaengine_unregister(struct ccp_device *ccp) 743 { 744 struct dma_device *dma_dev = &ccp->dma_dev; 745 746 dma_async_device_unregister(dma_dev); 747 748 kmem_cache_destroy(ccp->dma_desc_cache); 749 kmem_cache_destroy(ccp->dma_cmd_cache); 750 } 751