1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Cryptographic Coprocessor (CCP) driver 4 * 5 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. 6 * 7 * Author: Gary R Hook <gary.hook@amd.com> 8 */ 9 10 #include <linux/module.h> 11 #include <linux/kernel.h> 12 #include <linux/dmaengine.h> 13 #include <linux/spinlock.h> 14 #include <linux/mutex.h> 15 #include <linux/ccp.h> 16 17 #include "ccp-dev.h" 18 #include "../../dma/dmaengine.h" 19 20 #define CCP_DMA_WIDTH(_mask) \ 21 ({ \ 22 u64 mask = _mask + 1; \ 23 (mask == 0) ? 64 : fls64(mask); \ 24 }) 25 26 /* The CCP as a DMA provider can be configured for public or private 27 * channels. Default is specified in the vdata for the device (PCI ID). 28 * This module parameter will override for all channels on all devices: 29 * dma_chan_attr = 0x2 to force all channels public 30 * = 0x1 to force all channels private 31 * = 0x0 to defer to the vdata setting 32 * = any other value: warning, revert to 0x0 33 */ 34 static unsigned int dma_chan_attr = CCP_DMA_DFLT; 35 module_param(dma_chan_attr, uint, 0444); 36 MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); 37 38 static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) 39 { 40 switch (dma_chan_attr) { 41 case CCP_DMA_DFLT: 42 return ccp->vdata->dma_chan_attr; 43 44 case CCP_DMA_PRIV: 45 return DMA_PRIVATE; 46 47 case CCP_DMA_PUB: 48 return 0; 49 50 default: 51 dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", 52 dma_chan_attr); 53 return ccp->vdata->dma_chan_attr; 54 } 55 } 56 57 static void ccp_free_cmd_resources(struct ccp_device *ccp, 58 struct list_head *list) 59 { 60 struct ccp_dma_cmd *cmd, *ctmp; 61 62 list_for_each_entry_safe(cmd, ctmp, list, entry) { 63 list_del(&cmd->entry); 64 kmem_cache_free(ccp->dma_cmd_cache, cmd); 65 } 66 } 67 68 static void ccp_free_desc_resources(struct ccp_device *ccp, 69 struct list_head *list) 70 { 71 struct ccp_dma_desc *desc, *dtmp; 72 73 list_for_each_entry_safe(desc, dtmp, list, entry) { 74 ccp_free_cmd_resources(ccp, &desc->active); 75 ccp_free_cmd_resources(ccp, &desc->pending); 76 77 list_del(&desc->entry); 78 kmem_cache_free(ccp->dma_desc_cache, desc); 79 } 80 } 81 82 static void ccp_free_chan_resources(struct dma_chan *dma_chan) 83 { 84 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 85 dma_chan); 86 unsigned long flags; 87 88 dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); 89 90 spin_lock_irqsave(&chan->lock, flags); 91 92 ccp_free_desc_resources(chan->ccp, &chan->complete); 93 ccp_free_desc_resources(chan->ccp, &chan->active); 94 ccp_free_desc_resources(chan->ccp, &chan->pending); 95 ccp_free_desc_resources(chan->ccp, &chan->created); 96 97 spin_unlock_irqrestore(&chan->lock, flags); 98 } 99 100 static void ccp_cleanup_desc_resources(struct ccp_device *ccp, 101 struct list_head *list) 102 { 103 struct ccp_dma_desc *desc, *dtmp; 104 105 list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { 106 if (!async_tx_test_ack(&desc->tx_desc)) 107 continue; 108 109 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 110 111 ccp_free_cmd_resources(ccp, &desc->active); 112 ccp_free_cmd_resources(ccp, &desc->pending); 113 114 list_del(&desc->entry); 115 kmem_cache_free(ccp->dma_desc_cache, desc); 116 } 117 } 118 119 static void ccp_do_cleanup(unsigned long data) 120 { 121 struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; 122 unsigned long flags; 123 124 dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, 125 dma_chan_name(&chan->dma_chan)); 126 127 spin_lock_irqsave(&chan->lock, flags); 128 129 ccp_cleanup_desc_resources(chan->ccp, &chan->complete); 130 131 spin_unlock_irqrestore(&chan->lock, flags); 132 } 133 134 static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) 135 { 136 struct ccp_dma_cmd *cmd; 137 int ret; 138 139 cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); 140 list_move(&cmd->entry, &desc->active); 141 142 dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, 143 desc->tx_desc.cookie, cmd); 144 145 ret = ccp_enqueue_cmd(&cmd->ccp_cmd); 146 if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) 147 return 0; 148 149 dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, 150 ret, desc->tx_desc.cookie, cmd); 151 152 return ret; 153 } 154 155 static void ccp_free_active_cmd(struct ccp_dma_desc *desc) 156 { 157 struct ccp_dma_cmd *cmd; 158 159 cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, 160 entry); 161 if (!cmd) 162 return; 163 164 dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", 165 __func__, desc->tx_desc.cookie, cmd); 166 167 list_del(&cmd->entry); 168 kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); 169 } 170 171 static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, 172 struct ccp_dma_desc *desc) 173 { 174 /* Move current DMA descriptor to the complete list */ 175 if (desc) 176 list_move(&desc->entry, &chan->complete); 177 178 /* Get the next DMA descriptor on the active list */ 179 desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 180 entry); 181 182 return desc; 183 } 184 185 static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, 186 struct ccp_dma_desc *desc) 187 { 188 struct dma_async_tx_descriptor *tx_desc; 189 unsigned long flags; 190 191 /* Loop over descriptors until one is found with commands */ 192 do { 193 if (desc) { 194 /* Remove the DMA command from the list and free it */ 195 ccp_free_active_cmd(desc); 196 197 if (!list_empty(&desc->pending)) { 198 /* No errors, keep going */ 199 if (desc->status != DMA_ERROR) 200 return desc; 201 202 /* Error, free remaining commands and move on */ 203 ccp_free_cmd_resources(desc->ccp, 204 &desc->pending); 205 } 206 207 tx_desc = &desc->tx_desc; 208 } else { 209 tx_desc = NULL; 210 } 211 212 spin_lock_irqsave(&chan->lock, flags); 213 214 if (desc) { 215 if (desc->status != DMA_ERROR) 216 desc->status = DMA_COMPLETE; 217 218 dev_dbg(desc->ccp->dev, 219 "%s - tx %d complete, status=%u\n", __func__, 220 desc->tx_desc.cookie, desc->status); 221 222 dma_cookie_complete(tx_desc); 223 dma_descriptor_unmap(tx_desc); 224 } 225 226 desc = __ccp_next_dma_desc(chan, desc); 227 228 spin_unlock_irqrestore(&chan->lock, flags); 229 230 if (tx_desc) { 231 dmaengine_desc_get_callback_invoke(tx_desc, NULL); 232 233 dma_run_dependencies(tx_desc); 234 } 235 } while (desc); 236 237 return NULL; 238 } 239 240 static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) 241 { 242 struct ccp_dma_desc *desc; 243 244 if (list_empty(&chan->pending)) 245 return NULL; 246 247 desc = list_empty(&chan->active) 248 ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) 249 : NULL; 250 251 list_splice_tail_init(&chan->pending, &chan->active); 252 253 return desc; 254 } 255 256 static void ccp_cmd_callback(void *data, int err) 257 { 258 struct ccp_dma_desc *desc = data; 259 struct ccp_dma_chan *chan; 260 int ret; 261 262 if (err == -EINPROGRESS) 263 return; 264 265 chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, 266 dma_chan); 267 268 dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", 269 __func__, desc->tx_desc.cookie, err); 270 271 if (err) 272 desc->status = DMA_ERROR; 273 274 while (true) { 275 /* Check for DMA descriptor completion */ 276 desc = ccp_handle_active_desc(chan, desc); 277 278 /* Don't submit cmd if no descriptor or DMA is paused */ 279 if (!desc || (chan->status == DMA_PAUSED)) 280 break; 281 282 ret = ccp_issue_next_cmd(desc); 283 if (!ret) 284 break; 285 286 desc->status = DMA_ERROR; 287 } 288 289 tasklet_schedule(&chan->cleanup_tasklet); 290 } 291 292 static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) 293 { 294 struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, 295 tx_desc); 296 struct ccp_dma_chan *chan; 297 dma_cookie_t cookie; 298 unsigned long flags; 299 300 chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); 301 302 spin_lock_irqsave(&chan->lock, flags); 303 304 cookie = dma_cookie_assign(tx_desc); 305 list_del(&desc->entry); 306 list_add_tail(&desc->entry, &chan->pending); 307 308 spin_unlock_irqrestore(&chan->lock, flags); 309 310 dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", 311 __func__, cookie); 312 313 return cookie; 314 } 315 316 static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) 317 { 318 struct ccp_dma_cmd *cmd; 319 320 cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); 321 if (cmd) 322 memset(cmd, 0, sizeof(*cmd)); 323 324 return cmd; 325 } 326 327 static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, 328 unsigned long flags) 329 { 330 struct ccp_dma_desc *desc; 331 332 desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); 333 if (!desc) 334 return NULL; 335 336 dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); 337 desc->tx_desc.flags = flags; 338 desc->tx_desc.tx_submit = ccp_tx_submit; 339 desc->ccp = chan->ccp; 340 INIT_LIST_HEAD(&desc->pending); 341 INIT_LIST_HEAD(&desc->active); 342 desc->status = DMA_IN_PROGRESS; 343 344 return desc; 345 } 346 347 static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, 348 struct scatterlist *dst_sg, 349 unsigned int dst_nents, 350 struct scatterlist *src_sg, 351 unsigned int src_nents, 352 unsigned long flags) 353 { 354 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 355 dma_chan); 356 struct ccp_device *ccp = chan->ccp; 357 struct ccp_dma_desc *desc; 358 struct ccp_dma_cmd *cmd; 359 struct ccp_cmd *ccp_cmd; 360 struct ccp_passthru_nomap_engine *ccp_pt; 361 unsigned int src_offset, src_len; 362 unsigned int dst_offset, dst_len; 363 unsigned int len; 364 unsigned long sflags; 365 size_t total_len; 366 367 if (!dst_sg || !src_sg) 368 return NULL; 369 370 if (!dst_nents || !src_nents) 371 return NULL; 372 373 desc = ccp_alloc_dma_desc(chan, flags); 374 if (!desc) 375 return NULL; 376 377 total_len = 0; 378 379 src_len = sg_dma_len(src_sg); 380 src_offset = 0; 381 382 dst_len = sg_dma_len(dst_sg); 383 dst_offset = 0; 384 385 while (true) { 386 if (!src_len) { 387 src_nents--; 388 if (!src_nents) 389 break; 390 391 src_sg = sg_next(src_sg); 392 if (!src_sg) 393 break; 394 395 src_len = sg_dma_len(src_sg); 396 src_offset = 0; 397 continue; 398 } 399 400 if (!dst_len) { 401 dst_nents--; 402 if (!dst_nents) 403 break; 404 405 dst_sg = sg_next(dst_sg); 406 if (!dst_sg) 407 break; 408 409 dst_len = sg_dma_len(dst_sg); 410 dst_offset = 0; 411 continue; 412 } 413 414 len = min(dst_len, src_len); 415 416 cmd = ccp_alloc_dma_cmd(chan); 417 if (!cmd) 418 goto err; 419 420 ccp_cmd = &cmd->ccp_cmd; 421 ccp_cmd->ccp = chan->ccp; 422 ccp_pt = &ccp_cmd->u.passthru_nomap; 423 ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; 424 ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; 425 ccp_cmd->engine = CCP_ENGINE_PASSTHRU; 426 ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; 427 ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; 428 ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; 429 ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; 430 ccp_pt->src_len = len; 431 ccp_pt->final = 1; 432 ccp_cmd->callback = ccp_cmd_callback; 433 ccp_cmd->data = desc; 434 435 list_add_tail(&cmd->entry, &desc->pending); 436 437 dev_dbg(ccp->dev, 438 "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, 439 cmd, &ccp_pt->src_dma, 440 &ccp_pt->dst_dma, ccp_pt->src_len); 441 442 total_len += len; 443 444 src_len -= len; 445 src_offset += len; 446 447 dst_len -= len; 448 dst_offset += len; 449 } 450 451 desc->len = total_len; 452 453 if (list_empty(&desc->pending)) 454 goto err; 455 456 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); 457 458 spin_lock_irqsave(&chan->lock, sflags); 459 460 list_add_tail(&desc->entry, &chan->created); 461 462 spin_unlock_irqrestore(&chan->lock, sflags); 463 464 return desc; 465 466 err: 467 ccp_free_cmd_resources(ccp, &desc->pending); 468 kmem_cache_free(ccp->dma_desc_cache, desc); 469 470 return NULL; 471 } 472 473 static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( 474 struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, 475 unsigned long flags) 476 { 477 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 478 dma_chan); 479 struct ccp_dma_desc *desc; 480 struct scatterlist dst_sg, src_sg; 481 482 dev_dbg(chan->ccp->dev, 483 "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", 484 __func__, &src, &dst, len, flags); 485 486 sg_init_table(&dst_sg, 1); 487 sg_dma_address(&dst_sg) = dst; 488 sg_dma_len(&dst_sg) = len; 489 490 sg_init_table(&src_sg, 1); 491 sg_dma_address(&src_sg) = src; 492 sg_dma_len(&src_sg) = len; 493 494 desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); 495 if (!desc) 496 return NULL; 497 498 return &desc->tx_desc; 499 } 500 501 static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( 502 struct dma_chan *dma_chan, unsigned long flags) 503 { 504 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 505 dma_chan); 506 struct ccp_dma_desc *desc; 507 508 desc = ccp_alloc_dma_desc(chan, flags); 509 if (!desc) 510 return NULL; 511 512 return &desc->tx_desc; 513 } 514 515 static void ccp_issue_pending(struct dma_chan *dma_chan) 516 { 517 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 518 dma_chan); 519 struct ccp_dma_desc *desc; 520 unsigned long flags; 521 522 dev_dbg(chan->ccp->dev, "%s\n", __func__); 523 524 spin_lock_irqsave(&chan->lock, flags); 525 526 desc = __ccp_pending_to_active(chan); 527 528 spin_unlock_irqrestore(&chan->lock, flags); 529 530 /* If there was nothing active, start processing */ 531 if (desc) 532 ccp_cmd_callback(desc, 0); 533 } 534 535 static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, 536 dma_cookie_t cookie, 537 struct dma_tx_state *state) 538 { 539 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 540 dma_chan); 541 struct ccp_dma_desc *desc; 542 enum dma_status ret; 543 unsigned long flags; 544 545 if (chan->status == DMA_PAUSED) { 546 ret = DMA_PAUSED; 547 goto out; 548 } 549 550 ret = dma_cookie_status(dma_chan, cookie, state); 551 if (ret == DMA_COMPLETE) { 552 spin_lock_irqsave(&chan->lock, flags); 553 554 /* Get status from complete chain, if still there */ 555 list_for_each_entry(desc, &chan->complete, entry) { 556 if (desc->tx_desc.cookie != cookie) 557 continue; 558 559 ret = desc->status; 560 break; 561 } 562 563 spin_unlock_irqrestore(&chan->lock, flags); 564 } 565 566 out: 567 dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); 568 569 return ret; 570 } 571 572 static int ccp_pause(struct dma_chan *dma_chan) 573 { 574 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 575 dma_chan); 576 577 chan->status = DMA_PAUSED; 578 579 /*TODO: Wait for active DMA to complete before returning? */ 580 581 return 0; 582 } 583 584 static int ccp_resume(struct dma_chan *dma_chan) 585 { 586 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 587 dma_chan); 588 struct ccp_dma_desc *desc; 589 unsigned long flags; 590 591 spin_lock_irqsave(&chan->lock, flags); 592 593 desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, 594 entry); 595 596 spin_unlock_irqrestore(&chan->lock, flags); 597 598 /* Indicate the channel is running again */ 599 chan->status = DMA_IN_PROGRESS; 600 601 /* If there was something active, re-start */ 602 if (desc) 603 ccp_cmd_callback(desc, 0); 604 605 return 0; 606 } 607 608 static int ccp_terminate_all(struct dma_chan *dma_chan) 609 { 610 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, 611 dma_chan); 612 unsigned long flags; 613 614 dev_dbg(chan->ccp->dev, "%s\n", __func__); 615 616 /*TODO: Wait for active DMA to complete before continuing */ 617 618 spin_lock_irqsave(&chan->lock, flags); 619 620 /*TODO: Purge the complete list? */ 621 ccp_free_desc_resources(chan->ccp, &chan->active); 622 ccp_free_desc_resources(chan->ccp, &chan->pending); 623 ccp_free_desc_resources(chan->ccp, &chan->created); 624 625 spin_unlock_irqrestore(&chan->lock, flags); 626 627 return 0; 628 } 629 630 int ccp_dmaengine_register(struct ccp_device *ccp) 631 { 632 struct ccp_dma_chan *chan; 633 struct dma_device *dma_dev = &ccp->dma_dev; 634 struct dma_chan *dma_chan; 635 char *dma_cmd_cache_name; 636 char *dma_desc_cache_name; 637 unsigned int i; 638 int ret; 639 640 ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, 641 sizeof(*(ccp->ccp_dma_chan)), 642 GFP_KERNEL); 643 if (!ccp->ccp_dma_chan) 644 return -ENOMEM; 645 646 dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 647 "%s-dmaengine-cmd-cache", 648 ccp->name); 649 if (!dma_cmd_cache_name) 650 return -ENOMEM; 651 652 ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, 653 sizeof(struct ccp_dma_cmd), 654 sizeof(void *), 655 SLAB_HWCACHE_ALIGN, NULL); 656 if (!ccp->dma_cmd_cache) 657 return -ENOMEM; 658 659 dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, 660 "%s-dmaengine-desc-cache", 661 ccp->name); 662 if (!dma_desc_cache_name) { 663 ret = -ENOMEM; 664 goto err_cache; 665 } 666 667 ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, 668 sizeof(struct ccp_dma_desc), 669 sizeof(void *), 670 SLAB_HWCACHE_ALIGN, NULL); 671 if (!ccp->dma_desc_cache) { 672 ret = -ENOMEM; 673 goto err_cache; 674 } 675 676 dma_dev->dev = ccp->dev; 677 dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 678 dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); 679 dma_dev->directions = DMA_MEM_TO_MEM; 680 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 681 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 682 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 683 684 /* The DMA channels for this device can be set to public or private, 685 * and overridden by the module parameter dma_chan_attr. 686 * Default: according to the value in vdata (dma_chan_attr=0) 687 * dma_chan_attr=0x1: all channels private (override vdata) 688 * dma_chan_attr=0x2: all channels public (override vdata) 689 */ 690 if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) 691 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); 692 693 INIT_LIST_HEAD(&dma_dev->channels); 694 for (i = 0; i < ccp->cmd_q_count; i++) { 695 chan = ccp->ccp_dma_chan + i; 696 dma_chan = &chan->dma_chan; 697 698 chan->ccp = ccp; 699 700 spin_lock_init(&chan->lock); 701 INIT_LIST_HEAD(&chan->created); 702 INIT_LIST_HEAD(&chan->pending); 703 INIT_LIST_HEAD(&chan->active); 704 INIT_LIST_HEAD(&chan->complete); 705 706 tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, 707 (unsigned long)chan); 708 709 dma_chan->device = dma_dev; 710 dma_cookie_init(dma_chan); 711 712 list_add_tail(&dma_chan->device_node, &dma_dev->channels); 713 } 714 715 dma_dev->device_free_chan_resources = ccp_free_chan_resources; 716 dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; 717 dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; 718 dma_dev->device_issue_pending = ccp_issue_pending; 719 dma_dev->device_tx_status = ccp_tx_status; 720 dma_dev->device_pause = ccp_pause; 721 dma_dev->device_resume = ccp_resume; 722 dma_dev->device_terminate_all = ccp_terminate_all; 723 724 ret = dma_async_device_register(dma_dev); 725 if (ret) 726 goto err_reg; 727 728 return 0; 729 730 err_reg: 731 kmem_cache_destroy(ccp->dma_desc_cache); 732 733 err_cache: 734 kmem_cache_destroy(ccp->dma_cmd_cache); 735 736 return ret; 737 } 738 739 void ccp_dmaengine_unregister(struct ccp_device *ccp) 740 { 741 struct dma_device *dma_dev = &ccp->dma_dev; 742 743 dma_async_device_unregister(dma_dev); 744 745 kmem_cache_destroy(ccp->dma_desc_cache); 746 kmem_cache_destroy(ccp->dma_cmd_cache); 747 } 748