1 // SPDX-License-Identifier: GPL-2.0-only 2 // Copyright (C) 2017 Broadcom 3 4 /* 5 * Broadcom SBA RAID Driver 6 * 7 * The Broadcom stream buffer accelerator (SBA) provides offloading 8 * capabilities for RAID operations. The SBA offload engine is accessible 9 * via Broadcom SoC specific ring manager. Two or more offload engines 10 * can share same Broadcom SoC specific ring manager due to this Broadcom 11 * SoC specific ring manager driver is implemented as a mailbox controller 12 * driver and offload engine drivers are implemented as mallbox clients. 13 * 14 * Typically, Broadcom SoC specific ring manager will implement larger 15 * number of hardware rings over one or more SBA hardware devices. By 16 * design, the internal buffer size of SBA hardware device is limited 17 * but all offload operations supported by SBA can be broken down into 18 * multiple small size requests and executed parallely on multiple SBA 19 * hardware devices for achieving high through-put. 20 * 21 * The Broadcom SBA RAID driver does not require any register programming 22 * except submitting request to SBA hardware device via mailbox channels. 23 * This driver implements a DMA device with one DMA channel using a single 24 * mailbox channel provided by Broadcom SoC specific ring manager driver. 25 * For having more SBA DMA channels, we can create more SBA device nodes 26 * in Broadcom SoC specific DTS based on number of hardware rings supported 27 * by Broadcom SoC ring manager. 28 */ 29 30 #include <linux/bitops.h> 31 #include <linux/debugfs.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/dmaengine.h> 34 #include <linux/list.h> 35 #include <linux/mailbox_client.h> 36 #include <linux/mailbox/brcm-message.h> 37 #include <linux/module.h> 38 #include <linux/of.h> 39 #include <linux/of_platform.h> 40 #include <linux/platform_device.h> 41 #include <linux/slab.h> 42 #include <linux/raid/pq.h> 43 44 #include "dmaengine.h" 45 46 /* ====== Driver macros and defines ===== */ 47 48 #define SBA_TYPE_SHIFT 48 49 #define SBA_TYPE_MASK GENMASK(1, 0) 50 #define SBA_TYPE_A 0x0 51 #define SBA_TYPE_B 0x2 52 #define SBA_TYPE_C 0x3 53 #define SBA_USER_DEF_SHIFT 32 54 #define SBA_USER_DEF_MASK GENMASK(15, 0) 55 #define SBA_R_MDATA_SHIFT 24 56 #define SBA_R_MDATA_MASK GENMASK(7, 0) 57 #define SBA_C_MDATA_MS_SHIFT 18 58 #define SBA_C_MDATA_MS_MASK GENMASK(1, 0) 59 #define SBA_INT_SHIFT 17 60 #define SBA_INT_MASK BIT(0) 61 #define SBA_RESP_SHIFT 16 62 #define SBA_RESP_MASK BIT(0) 63 #define SBA_C_MDATA_SHIFT 8 64 #define SBA_C_MDATA_MASK GENMASK(7, 0) 65 #define SBA_C_MDATA_BNUMx_SHIFT(__bnum) (2 * (__bnum)) 66 #define SBA_C_MDATA_BNUMx_MASK GENMASK(1, 0) 67 #define SBA_C_MDATA_DNUM_SHIFT 5 68 #define SBA_C_MDATA_DNUM_MASK GENMASK(4, 0) 69 #define SBA_C_MDATA_LS(__v) ((__v) & 0xff) 70 #define SBA_C_MDATA_MS(__v) (((__v) >> 8) & 0x3) 71 #define SBA_CMD_SHIFT 0 72 #define SBA_CMD_MASK GENMASK(3, 0) 73 #define SBA_CMD_ZERO_BUFFER 0x4 74 #define SBA_CMD_ZERO_ALL_BUFFERS 0x8 75 #define SBA_CMD_LOAD_BUFFER 0x9 76 #define SBA_CMD_XOR 0xa 77 #define SBA_CMD_GALOIS_XOR 0xb 78 #define SBA_CMD_WRITE_BUFFER 0xc 79 #define SBA_CMD_GALOIS 0xe 80 81 #define SBA_MAX_REQ_PER_MBOX_CHANNEL 8192 82 #define SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL 8 83 84 /* Driver helper macros */ 85 #define to_sba_request(tx) \ 86 container_of(tx, struct sba_request, tx) 87 #define to_sba_device(dchan) \ 88 container_of(dchan, struct sba_device, dma_chan) 89 90 /* ===== Driver data structures ===== */ 91 92 enum sba_request_flags { 93 SBA_REQUEST_STATE_FREE = 0x001, 94 SBA_REQUEST_STATE_ALLOCED = 0x002, 95 SBA_REQUEST_STATE_PENDING = 0x004, 96 SBA_REQUEST_STATE_ACTIVE = 0x008, 97 SBA_REQUEST_STATE_ABORTED = 0x010, 98 SBA_REQUEST_STATE_MASK = 0x0ff, 99 SBA_REQUEST_FENCE = 0x100, 100 }; 101 102 struct sba_request { 103 /* Global state */ 104 struct list_head node; 105 struct sba_device *sba; 106 u32 flags; 107 /* Chained requests management */ 108 struct sba_request *first; 109 struct list_head next; 110 atomic_t next_pending_count; 111 /* BRCM message data */ 112 struct brcm_message msg; 113 struct dma_async_tx_descriptor tx; 114 /* SBA commands */ 115 struct brcm_sba_command cmds[]; 116 }; 117 118 enum sba_version { 119 SBA_VER_1 = 0, 120 SBA_VER_2 121 }; 122 123 struct sba_device { 124 /* Underlying device */ 125 struct device *dev; 126 /* DT configuration parameters */ 127 enum sba_version ver; 128 /* Derived configuration parameters */ 129 u32 max_req; 130 u32 hw_buf_size; 131 u32 hw_resp_size; 132 u32 max_pq_coefs; 133 u32 max_pq_srcs; 134 u32 max_cmd_per_req; 135 u32 max_xor_srcs; 136 u32 max_resp_pool_size; 137 u32 max_cmds_pool_size; 138 /* Maibox client and Mailbox channels */ 139 struct mbox_client client; 140 struct mbox_chan *mchan; 141 struct device *mbox_dev; 142 /* DMA device and DMA channel */ 143 struct dma_device dma_dev; 144 struct dma_chan dma_chan; 145 /* DMA channel resources */ 146 void *resp_base; 147 dma_addr_t resp_dma_base; 148 void *cmds_base; 149 dma_addr_t cmds_dma_base; 150 spinlock_t reqs_lock; 151 bool reqs_fence; 152 struct list_head reqs_alloc_list; 153 struct list_head reqs_pending_list; 154 struct list_head reqs_active_list; 155 struct list_head reqs_aborted_list; 156 struct list_head reqs_free_list; 157 /* DebugFS directory entries */ 158 struct dentry *root; 159 }; 160 161 /* ====== Command helper routines ===== */ 162 163 static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask) 164 { 165 cmd &= ~((u64)mask << shift); 166 cmd |= ((u64)(val & mask) << shift); 167 return cmd; 168 } 169 170 static inline u32 __pure sba_cmd_load_c_mdata(u32 b0) 171 { 172 return b0 & SBA_C_MDATA_BNUMx_MASK; 173 } 174 175 static inline u32 __pure sba_cmd_write_c_mdata(u32 b0) 176 { 177 return b0 & SBA_C_MDATA_BNUMx_MASK; 178 } 179 180 static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0) 181 { 182 return (b0 & SBA_C_MDATA_BNUMx_MASK) | 183 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)); 184 } 185 186 static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0) 187 { 188 return (b0 & SBA_C_MDATA_BNUMx_MASK) | 189 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) | 190 ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT); 191 } 192 193 /* ====== General helper routines ===== */ 194 195 static struct sba_request *sba_alloc_request(struct sba_device *sba) 196 { 197 bool found = false; 198 unsigned long flags; 199 struct sba_request *req = NULL; 200 201 spin_lock_irqsave(&sba->reqs_lock, flags); 202 list_for_each_entry(req, &sba->reqs_free_list, node) { 203 if (async_tx_test_ack(&req->tx)) { 204 list_move_tail(&req->node, &sba->reqs_alloc_list); 205 found = true; 206 break; 207 } 208 } 209 spin_unlock_irqrestore(&sba->reqs_lock, flags); 210 211 if (!found) { 212 /* 213 * We have no more free requests so, we peek 214 * mailbox channels hoping few active requests 215 * would have completed which will create more 216 * room for new requests. 217 */ 218 mbox_client_peek_data(sba->mchan); 219 return NULL; 220 } 221 222 req->flags = SBA_REQUEST_STATE_ALLOCED; 223 req->first = req; 224 INIT_LIST_HEAD(&req->next); 225 atomic_set(&req->next_pending_count, 1); 226 227 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); 228 async_tx_ack(&req->tx); 229 230 return req; 231 } 232 233 /* Note: Must be called with sba->reqs_lock held */ 234 static void _sba_pending_request(struct sba_device *sba, 235 struct sba_request *req) 236 { 237 lockdep_assert_held(&sba->reqs_lock); 238 req->flags &= ~SBA_REQUEST_STATE_MASK; 239 req->flags |= SBA_REQUEST_STATE_PENDING; 240 list_move_tail(&req->node, &sba->reqs_pending_list); 241 if (list_empty(&sba->reqs_active_list)) 242 sba->reqs_fence = false; 243 } 244 245 /* Note: Must be called with sba->reqs_lock held */ 246 static bool _sba_active_request(struct sba_device *sba, 247 struct sba_request *req) 248 { 249 lockdep_assert_held(&sba->reqs_lock); 250 if (list_empty(&sba->reqs_active_list)) 251 sba->reqs_fence = false; 252 if (sba->reqs_fence) 253 return false; 254 req->flags &= ~SBA_REQUEST_STATE_MASK; 255 req->flags |= SBA_REQUEST_STATE_ACTIVE; 256 list_move_tail(&req->node, &sba->reqs_active_list); 257 if (req->flags & SBA_REQUEST_FENCE) 258 sba->reqs_fence = true; 259 return true; 260 } 261 262 /* Note: Must be called with sba->reqs_lock held */ 263 static void _sba_abort_request(struct sba_device *sba, 264 struct sba_request *req) 265 { 266 lockdep_assert_held(&sba->reqs_lock); 267 req->flags &= ~SBA_REQUEST_STATE_MASK; 268 req->flags |= SBA_REQUEST_STATE_ABORTED; 269 list_move_tail(&req->node, &sba->reqs_aborted_list); 270 if (list_empty(&sba->reqs_active_list)) 271 sba->reqs_fence = false; 272 } 273 274 /* Note: Must be called with sba->reqs_lock held */ 275 static void _sba_free_request(struct sba_device *sba, 276 struct sba_request *req) 277 { 278 lockdep_assert_held(&sba->reqs_lock); 279 req->flags &= ~SBA_REQUEST_STATE_MASK; 280 req->flags |= SBA_REQUEST_STATE_FREE; 281 list_move_tail(&req->node, &sba->reqs_free_list); 282 if (list_empty(&sba->reqs_active_list)) 283 sba->reqs_fence = false; 284 } 285 286 static void sba_free_chained_requests(struct sba_request *req) 287 { 288 unsigned long flags; 289 struct sba_request *nreq; 290 struct sba_device *sba = req->sba; 291 292 spin_lock_irqsave(&sba->reqs_lock, flags); 293 294 _sba_free_request(sba, req); 295 list_for_each_entry(nreq, &req->next, next) 296 _sba_free_request(sba, nreq); 297 298 spin_unlock_irqrestore(&sba->reqs_lock, flags); 299 } 300 301 static void sba_chain_request(struct sba_request *first, 302 struct sba_request *req) 303 { 304 unsigned long flags; 305 struct sba_device *sba = req->sba; 306 307 spin_lock_irqsave(&sba->reqs_lock, flags); 308 309 list_add_tail(&req->next, &first->next); 310 req->first = first; 311 atomic_inc(&first->next_pending_count); 312 313 spin_unlock_irqrestore(&sba->reqs_lock, flags); 314 } 315 316 static void sba_cleanup_nonpending_requests(struct sba_device *sba) 317 { 318 unsigned long flags; 319 struct sba_request *req, *req1; 320 321 spin_lock_irqsave(&sba->reqs_lock, flags); 322 323 /* Freeup all alloced request */ 324 list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node) 325 _sba_free_request(sba, req); 326 327 /* Set all active requests as aborted */ 328 list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node) 329 _sba_abort_request(sba, req); 330 331 /* 332 * Note: We expect that aborted request will be eventually 333 * freed by sba_receive_message() 334 */ 335 336 spin_unlock_irqrestore(&sba->reqs_lock, flags); 337 } 338 339 static void sba_cleanup_pending_requests(struct sba_device *sba) 340 { 341 unsigned long flags; 342 struct sba_request *req, *req1; 343 344 spin_lock_irqsave(&sba->reqs_lock, flags); 345 346 /* Freeup all pending request */ 347 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) 348 _sba_free_request(sba, req); 349 350 spin_unlock_irqrestore(&sba->reqs_lock, flags); 351 } 352 353 static int sba_send_mbox_request(struct sba_device *sba, 354 struct sba_request *req) 355 { 356 int ret = 0; 357 358 /* Send message for the request */ 359 req->msg.error = 0; 360 ret = mbox_send_message(sba->mchan, &req->msg); 361 if (ret < 0) { 362 dev_err(sba->dev, "send message failed with error %d", ret); 363 return ret; 364 } 365 366 /* Check error returned by mailbox controller */ 367 ret = req->msg.error; 368 if (ret < 0) { 369 dev_err(sba->dev, "message error %d", ret); 370 } 371 372 /* Signal txdone for mailbox channel */ 373 mbox_client_txdone(sba->mchan, ret); 374 375 return ret; 376 } 377 378 /* Note: Must be called with sba->reqs_lock held */ 379 static void _sba_process_pending_requests(struct sba_device *sba) 380 { 381 int ret; 382 u32 count; 383 struct sba_request *req; 384 385 /* Process few pending requests */ 386 count = SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL; 387 while (!list_empty(&sba->reqs_pending_list) && count) { 388 /* Get the first pending request */ 389 req = list_first_entry(&sba->reqs_pending_list, 390 struct sba_request, node); 391 392 /* Try to make request active */ 393 if (!_sba_active_request(sba, req)) 394 break; 395 396 /* Send request to mailbox channel */ 397 ret = sba_send_mbox_request(sba, req); 398 if (ret < 0) { 399 _sba_pending_request(sba, req); 400 break; 401 } 402 403 count--; 404 } 405 } 406 407 static void sba_process_received_request(struct sba_device *sba, 408 struct sba_request *req) 409 { 410 unsigned long flags; 411 struct dma_async_tx_descriptor *tx; 412 struct sba_request *nreq, *first = req->first; 413 414 /* Process only after all chained requests are received */ 415 if (!atomic_dec_return(&first->next_pending_count)) { 416 tx = &first->tx; 417 418 WARN_ON(tx->cookie < 0); 419 if (tx->cookie > 0) { 420 spin_lock_irqsave(&sba->reqs_lock, flags); 421 dma_cookie_complete(tx); 422 spin_unlock_irqrestore(&sba->reqs_lock, flags); 423 dmaengine_desc_get_callback_invoke(tx, NULL); 424 dma_descriptor_unmap(tx); 425 tx->callback = NULL; 426 tx->callback_result = NULL; 427 } 428 429 dma_run_dependencies(tx); 430 431 spin_lock_irqsave(&sba->reqs_lock, flags); 432 433 /* Free all requests chained to first request */ 434 list_for_each_entry(nreq, &first->next, next) 435 _sba_free_request(sba, nreq); 436 INIT_LIST_HEAD(&first->next); 437 438 /* Free the first request */ 439 _sba_free_request(sba, first); 440 441 /* Process pending requests */ 442 _sba_process_pending_requests(sba); 443 444 spin_unlock_irqrestore(&sba->reqs_lock, flags); 445 } 446 } 447 448 static void sba_write_stats_in_seqfile(struct sba_device *sba, 449 struct seq_file *file) 450 { 451 unsigned long flags; 452 struct sba_request *req; 453 u32 free_count = 0, alloced_count = 0; 454 u32 pending_count = 0, active_count = 0, aborted_count = 0; 455 456 spin_lock_irqsave(&sba->reqs_lock, flags); 457 458 list_for_each_entry(req, &sba->reqs_free_list, node) 459 if (async_tx_test_ack(&req->tx)) 460 free_count++; 461 462 list_for_each_entry(req, &sba->reqs_alloc_list, node) 463 alloced_count++; 464 465 list_for_each_entry(req, &sba->reqs_pending_list, node) 466 pending_count++; 467 468 list_for_each_entry(req, &sba->reqs_active_list, node) 469 active_count++; 470 471 list_for_each_entry(req, &sba->reqs_aborted_list, node) 472 aborted_count++; 473 474 spin_unlock_irqrestore(&sba->reqs_lock, flags); 475 476 seq_printf(file, "maximum requests = %d\n", sba->max_req); 477 seq_printf(file, "free requests = %d\n", free_count); 478 seq_printf(file, "alloced requests = %d\n", alloced_count); 479 seq_printf(file, "pending requests = %d\n", pending_count); 480 seq_printf(file, "active requests = %d\n", active_count); 481 seq_printf(file, "aborted requests = %d\n", aborted_count); 482 } 483 484 /* ====== DMAENGINE callbacks ===== */ 485 486 static void sba_free_chan_resources(struct dma_chan *dchan) 487 { 488 /* 489 * Channel resources are pre-alloced so we just free-up 490 * whatever we can so that we can re-use pre-alloced 491 * channel resources next time. 492 */ 493 sba_cleanup_nonpending_requests(to_sba_device(dchan)); 494 } 495 496 static int sba_device_terminate_all(struct dma_chan *dchan) 497 { 498 /* Cleanup all pending requests */ 499 sba_cleanup_pending_requests(to_sba_device(dchan)); 500 501 return 0; 502 } 503 504 static void sba_issue_pending(struct dma_chan *dchan) 505 { 506 unsigned long flags; 507 struct sba_device *sba = to_sba_device(dchan); 508 509 /* Process pending requests */ 510 spin_lock_irqsave(&sba->reqs_lock, flags); 511 _sba_process_pending_requests(sba); 512 spin_unlock_irqrestore(&sba->reqs_lock, flags); 513 } 514 515 static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx) 516 { 517 unsigned long flags; 518 dma_cookie_t cookie; 519 struct sba_device *sba; 520 struct sba_request *req, *nreq; 521 522 if (unlikely(!tx)) 523 return -EINVAL; 524 525 sba = to_sba_device(tx->chan); 526 req = to_sba_request(tx); 527 528 /* Assign cookie and mark all chained requests pending */ 529 spin_lock_irqsave(&sba->reqs_lock, flags); 530 cookie = dma_cookie_assign(tx); 531 _sba_pending_request(sba, req); 532 list_for_each_entry(nreq, &req->next, next) 533 _sba_pending_request(sba, nreq); 534 spin_unlock_irqrestore(&sba->reqs_lock, flags); 535 536 return cookie; 537 } 538 539 static enum dma_status sba_tx_status(struct dma_chan *dchan, 540 dma_cookie_t cookie, 541 struct dma_tx_state *txstate) 542 { 543 enum dma_status ret; 544 struct sba_device *sba = to_sba_device(dchan); 545 546 ret = dma_cookie_status(dchan, cookie, txstate); 547 if (ret == DMA_COMPLETE) 548 return ret; 549 550 mbox_client_peek_data(sba->mchan); 551 552 return dma_cookie_status(dchan, cookie, txstate); 553 } 554 555 static void sba_fillup_interrupt_msg(struct sba_request *req, 556 struct brcm_sba_command *cmds, 557 struct brcm_message *msg) 558 { 559 u64 cmd; 560 u32 c_mdata; 561 dma_addr_t resp_dma = req->tx.phys; 562 struct brcm_sba_command *cmdsp = cmds; 563 564 /* Type-B command to load dummy data into buf0 */ 565 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 566 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 567 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size, 568 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 569 c_mdata = sba_cmd_load_c_mdata(0); 570 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 571 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 572 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 573 SBA_CMD_SHIFT, SBA_CMD_MASK); 574 cmdsp->cmd = cmd; 575 *cmdsp->cmd_dma = cpu_to_le64(cmd); 576 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 577 cmdsp->data = resp_dma; 578 cmdsp->data_len = req->sba->hw_resp_size; 579 cmdsp++; 580 581 /* Type-A command to write buf0 to dummy location */ 582 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 583 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 584 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size, 585 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 586 cmd = sba_cmd_enc(cmd, 0x1, 587 SBA_RESP_SHIFT, SBA_RESP_MASK); 588 c_mdata = sba_cmd_write_c_mdata(0); 589 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 590 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 591 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 592 SBA_CMD_SHIFT, SBA_CMD_MASK); 593 cmdsp->cmd = cmd; 594 *cmdsp->cmd_dma = cpu_to_le64(cmd); 595 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 596 if (req->sba->hw_resp_size) { 597 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 598 cmdsp->resp = resp_dma; 599 cmdsp->resp_len = req->sba->hw_resp_size; 600 } 601 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 602 cmdsp->data = resp_dma; 603 cmdsp->data_len = req->sba->hw_resp_size; 604 cmdsp++; 605 606 /* Fillup brcm_message */ 607 msg->type = BRCM_MESSAGE_SBA; 608 msg->sba.cmds = cmds; 609 msg->sba.cmds_count = cmdsp - cmds; 610 msg->ctx = req; 611 msg->error = 0; 612 } 613 614 static struct dma_async_tx_descriptor * 615 sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags) 616 { 617 struct sba_request *req = NULL; 618 struct sba_device *sba = to_sba_device(dchan); 619 620 /* Alloc new request */ 621 req = sba_alloc_request(sba); 622 if (!req) 623 return NULL; 624 625 /* 626 * Force fence so that no requests are submitted 627 * until DMA callback for this request is invoked. 628 */ 629 req->flags |= SBA_REQUEST_FENCE; 630 631 /* Fillup request message */ 632 sba_fillup_interrupt_msg(req, req->cmds, &req->msg); 633 634 /* Init async_tx descriptor */ 635 req->tx.flags = flags; 636 req->tx.cookie = -EBUSY; 637 638 return &req->tx; 639 } 640 641 static void sba_fillup_memcpy_msg(struct sba_request *req, 642 struct brcm_sba_command *cmds, 643 struct brcm_message *msg, 644 dma_addr_t msg_offset, size_t msg_len, 645 dma_addr_t dst, dma_addr_t src) 646 { 647 u64 cmd; 648 u32 c_mdata; 649 dma_addr_t resp_dma = req->tx.phys; 650 struct brcm_sba_command *cmdsp = cmds; 651 652 /* Type-B command to load data into buf0 */ 653 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 654 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 655 cmd = sba_cmd_enc(cmd, msg_len, 656 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 657 c_mdata = sba_cmd_load_c_mdata(0); 658 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 659 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 660 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 661 SBA_CMD_SHIFT, SBA_CMD_MASK); 662 cmdsp->cmd = cmd; 663 *cmdsp->cmd_dma = cpu_to_le64(cmd); 664 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 665 cmdsp->data = src + msg_offset; 666 cmdsp->data_len = msg_len; 667 cmdsp++; 668 669 /* Type-A command to write buf0 */ 670 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 671 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 672 cmd = sba_cmd_enc(cmd, msg_len, 673 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 674 cmd = sba_cmd_enc(cmd, 0x1, 675 SBA_RESP_SHIFT, SBA_RESP_MASK); 676 c_mdata = sba_cmd_write_c_mdata(0); 677 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 678 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 679 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 680 SBA_CMD_SHIFT, SBA_CMD_MASK); 681 cmdsp->cmd = cmd; 682 *cmdsp->cmd_dma = cpu_to_le64(cmd); 683 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 684 if (req->sba->hw_resp_size) { 685 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 686 cmdsp->resp = resp_dma; 687 cmdsp->resp_len = req->sba->hw_resp_size; 688 } 689 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 690 cmdsp->data = dst + msg_offset; 691 cmdsp->data_len = msg_len; 692 cmdsp++; 693 694 /* Fillup brcm_message */ 695 msg->type = BRCM_MESSAGE_SBA; 696 msg->sba.cmds = cmds; 697 msg->sba.cmds_count = cmdsp - cmds; 698 msg->ctx = req; 699 msg->error = 0; 700 } 701 702 static struct sba_request * 703 sba_prep_dma_memcpy_req(struct sba_device *sba, 704 dma_addr_t off, dma_addr_t dst, dma_addr_t src, 705 size_t len, unsigned long flags) 706 { 707 struct sba_request *req = NULL; 708 709 /* Alloc new request */ 710 req = sba_alloc_request(sba); 711 if (!req) 712 return NULL; 713 if (flags & DMA_PREP_FENCE) 714 req->flags |= SBA_REQUEST_FENCE; 715 716 /* Fillup request message */ 717 sba_fillup_memcpy_msg(req, req->cmds, &req->msg, 718 off, len, dst, src); 719 720 /* Init async_tx descriptor */ 721 req->tx.flags = flags; 722 req->tx.cookie = -EBUSY; 723 724 return req; 725 } 726 727 static struct dma_async_tx_descriptor * 728 sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src, 729 size_t len, unsigned long flags) 730 { 731 size_t req_len; 732 dma_addr_t off = 0; 733 struct sba_device *sba = to_sba_device(dchan); 734 struct sba_request *first = NULL, *req; 735 736 /* Create chained requests where each request is upto hw_buf_size */ 737 while (len) { 738 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 739 740 req = sba_prep_dma_memcpy_req(sba, off, dst, src, 741 req_len, flags); 742 if (!req) { 743 if (first) 744 sba_free_chained_requests(first); 745 return NULL; 746 } 747 748 if (first) 749 sba_chain_request(first, req); 750 else 751 first = req; 752 753 off += req_len; 754 len -= req_len; 755 } 756 757 return (first) ? &first->tx : NULL; 758 } 759 760 static void sba_fillup_xor_msg(struct sba_request *req, 761 struct brcm_sba_command *cmds, 762 struct brcm_message *msg, 763 dma_addr_t msg_offset, size_t msg_len, 764 dma_addr_t dst, dma_addr_t *src, u32 src_cnt) 765 { 766 u64 cmd; 767 u32 c_mdata; 768 unsigned int i; 769 dma_addr_t resp_dma = req->tx.phys; 770 struct brcm_sba_command *cmdsp = cmds; 771 772 /* Type-B command to load data into buf0 */ 773 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 774 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 775 cmd = sba_cmd_enc(cmd, msg_len, 776 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 777 c_mdata = sba_cmd_load_c_mdata(0); 778 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 779 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 780 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 781 SBA_CMD_SHIFT, SBA_CMD_MASK); 782 cmdsp->cmd = cmd; 783 *cmdsp->cmd_dma = cpu_to_le64(cmd); 784 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 785 cmdsp->data = src[0] + msg_offset; 786 cmdsp->data_len = msg_len; 787 cmdsp++; 788 789 /* Type-B commands to xor data with buf0 and put it back in buf0 */ 790 for (i = 1; i < src_cnt; i++) { 791 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 792 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 793 cmd = sba_cmd_enc(cmd, msg_len, 794 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 795 c_mdata = sba_cmd_xor_c_mdata(0, 0); 796 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 797 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 798 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 799 SBA_CMD_SHIFT, SBA_CMD_MASK); 800 cmdsp->cmd = cmd; 801 *cmdsp->cmd_dma = cpu_to_le64(cmd); 802 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 803 cmdsp->data = src[i] + msg_offset; 804 cmdsp->data_len = msg_len; 805 cmdsp++; 806 } 807 808 /* Type-A command to write buf0 */ 809 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 810 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 811 cmd = sba_cmd_enc(cmd, msg_len, 812 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 813 cmd = sba_cmd_enc(cmd, 0x1, 814 SBA_RESP_SHIFT, SBA_RESP_MASK); 815 c_mdata = sba_cmd_write_c_mdata(0); 816 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 817 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 818 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 819 SBA_CMD_SHIFT, SBA_CMD_MASK); 820 cmdsp->cmd = cmd; 821 *cmdsp->cmd_dma = cpu_to_le64(cmd); 822 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 823 if (req->sba->hw_resp_size) { 824 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 825 cmdsp->resp = resp_dma; 826 cmdsp->resp_len = req->sba->hw_resp_size; 827 } 828 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 829 cmdsp->data = dst + msg_offset; 830 cmdsp->data_len = msg_len; 831 cmdsp++; 832 833 /* Fillup brcm_message */ 834 msg->type = BRCM_MESSAGE_SBA; 835 msg->sba.cmds = cmds; 836 msg->sba.cmds_count = cmdsp - cmds; 837 msg->ctx = req; 838 msg->error = 0; 839 } 840 841 static struct sba_request * 842 sba_prep_dma_xor_req(struct sba_device *sba, 843 dma_addr_t off, dma_addr_t dst, dma_addr_t *src, 844 u32 src_cnt, size_t len, unsigned long flags) 845 { 846 struct sba_request *req = NULL; 847 848 /* Alloc new request */ 849 req = sba_alloc_request(sba); 850 if (!req) 851 return NULL; 852 if (flags & DMA_PREP_FENCE) 853 req->flags |= SBA_REQUEST_FENCE; 854 855 /* Fillup request message */ 856 sba_fillup_xor_msg(req, req->cmds, &req->msg, 857 off, len, dst, src, src_cnt); 858 859 /* Init async_tx descriptor */ 860 req->tx.flags = flags; 861 req->tx.cookie = -EBUSY; 862 863 return req; 864 } 865 866 static struct dma_async_tx_descriptor * 867 sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src, 868 u32 src_cnt, size_t len, unsigned long flags) 869 { 870 size_t req_len; 871 dma_addr_t off = 0; 872 struct sba_device *sba = to_sba_device(dchan); 873 struct sba_request *first = NULL, *req; 874 875 /* Sanity checks */ 876 if (unlikely(src_cnt > sba->max_xor_srcs)) 877 return NULL; 878 879 /* Create chained requests where each request is upto hw_buf_size */ 880 while (len) { 881 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 882 883 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt, 884 req_len, flags); 885 if (!req) { 886 if (first) 887 sba_free_chained_requests(first); 888 return NULL; 889 } 890 891 if (first) 892 sba_chain_request(first, req); 893 else 894 first = req; 895 896 off += req_len; 897 len -= req_len; 898 } 899 900 return (first) ? &first->tx : NULL; 901 } 902 903 static void sba_fillup_pq_msg(struct sba_request *req, 904 bool pq_continue, 905 struct brcm_sba_command *cmds, 906 struct brcm_message *msg, 907 dma_addr_t msg_offset, size_t msg_len, 908 dma_addr_t *dst_p, dma_addr_t *dst_q, 909 const u8 *scf, dma_addr_t *src, u32 src_cnt) 910 { 911 u64 cmd; 912 u32 c_mdata; 913 unsigned int i; 914 dma_addr_t resp_dma = req->tx.phys; 915 struct brcm_sba_command *cmdsp = cmds; 916 917 if (pq_continue) { 918 /* Type-B command to load old P into buf0 */ 919 if (dst_p) { 920 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 921 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 922 cmd = sba_cmd_enc(cmd, msg_len, 923 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 924 c_mdata = sba_cmd_load_c_mdata(0); 925 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 926 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 927 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 928 SBA_CMD_SHIFT, SBA_CMD_MASK); 929 cmdsp->cmd = cmd; 930 *cmdsp->cmd_dma = cpu_to_le64(cmd); 931 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 932 cmdsp->data = *dst_p + msg_offset; 933 cmdsp->data_len = msg_len; 934 cmdsp++; 935 } 936 937 /* Type-B command to load old Q into buf1 */ 938 if (dst_q) { 939 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 940 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 941 cmd = sba_cmd_enc(cmd, msg_len, 942 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 943 c_mdata = sba_cmd_load_c_mdata(1); 944 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 945 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 946 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 947 SBA_CMD_SHIFT, SBA_CMD_MASK); 948 cmdsp->cmd = cmd; 949 *cmdsp->cmd_dma = cpu_to_le64(cmd); 950 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 951 cmdsp->data = *dst_q + msg_offset; 952 cmdsp->data_len = msg_len; 953 cmdsp++; 954 } 955 } else { 956 /* Type-A command to zero all buffers */ 957 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 958 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 959 cmd = sba_cmd_enc(cmd, msg_len, 960 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 961 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, 962 SBA_CMD_SHIFT, SBA_CMD_MASK); 963 cmdsp->cmd = cmd; 964 *cmdsp->cmd_dma = cpu_to_le64(cmd); 965 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 966 cmdsp++; 967 } 968 969 /* Type-B commands for generate P onto buf0 and Q onto buf1 */ 970 for (i = 0; i < src_cnt; i++) { 971 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 972 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 973 cmd = sba_cmd_enc(cmd, msg_len, 974 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 975 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0); 976 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 977 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 978 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 979 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 980 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR, 981 SBA_CMD_SHIFT, SBA_CMD_MASK); 982 cmdsp->cmd = cmd; 983 *cmdsp->cmd_dma = cpu_to_le64(cmd); 984 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 985 cmdsp->data = src[i] + msg_offset; 986 cmdsp->data_len = msg_len; 987 cmdsp++; 988 } 989 990 /* Type-A command to write buf0 */ 991 if (dst_p) { 992 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 993 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 994 cmd = sba_cmd_enc(cmd, msg_len, 995 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 996 cmd = sba_cmd_enc(cmd, 0x1, 997 SBA_RESP_SHIFT, SBA_RESP_MASK); 998 c_mdata = sba_cmd_write_c_mdata(0); 999 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1000 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1001 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1002 SBA_CMD_SHIFT, SBA_CMD_MASK); 1003 cmdsp->cmd = cmd; 1004 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1005 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1006 if (req->sba->hw_resp_size) { 1007 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1008 cmdsp->resp = resp_dma; 1009 cmdsp->resp_len = req->sba->hw_resp_size; 1010 } 1011 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1012 cmdsp->data = *dst_p + msg_offset; 1013 cmdsp->data_len = msg_len; 1014 cmdsp++; 1015 } 1016 1017 /* Type-A command to write buf1 */ 1018 if (dst_q) { 1019 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1020 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1021 cmd = sba_cmd_enc(cmd, msg_len, 1022 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1023 cmd = sba_cmd_enc(cmd, 0x1, 1024 SBA_RESP_SHIFT, SBA_RESP_MASK); 1025 c_mdata = sba_cmd_write_c_mdata(1); 1026 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1027 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1028 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1029 SBA_CMD_SHIFT, SBA_CMD_MASK); 1030 cmdsp->cmd = cmd; 1031 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1032 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1033 if (req->sba->hw_resp_size) { 1034 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1035 cmdsp->resp = resp_dma; 1036 cmdsp->resp_len = req->sba->hw_resp_size; 1037 } 1038 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1039 cmdsp->data = *dst_q + msg_offset; 1040 cmdsp->data_len = msg_len; 1041 cmdsp++; 1042 } 1043 1044 /* Fillup brcm_message */ 1045 msg->type = BRCM_MESSAGE_SBA; 1046 msg->sba.cmds = cmds; 1047 msg->sba.cmds_count = cmdsp - cmds; 1048 msg->ctx = req; 1049 msg->error = 0; 1050 } 1051 1052 static struct sba_request * 1053 sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off, 1054 dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src, 1055 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) 1056 { 1057 struct sba_request *req = NULL; 1058 1059 /* Alloc new request */ 1060 req = sba_alloc_request(sba); 1061 if (!req) 1062 return NULL; 1063 if (flags & DMA_PREP_FENCE) 1064 req->flags |= SBA_REQUEST_FENCE; 1065 1066 /* Fillup request messages */ 1067 sba_fillup_pq_msg(req, dmaf_continue(flags), 1068 req->cmds, &req->msg, 1069 off, len, dst_p, dst_q, scf, src, src_cnt); 1070 1071 /* Init async_tx descriptor */ 1072 req->tx.flags = flags; 1073 req->tx.cookie = -EBUSY; 1074 1075 return req; 1076 } 1077 1078 static void sba_fillup_pq_single_msg(struct sba_request *req, 1079 bool pq_continue, 1080 struct brcm_sba_command *cmds, 1081 struct brcm_message *msg, 1082 dma_addr_t msg_offset, size_t msg_len, 1083 dma_addr_t *dst_p, dma_addr_t *dst_q, 1084 dma_addr_t src, u8 scf) 1085 { 1086 u64 cmd; 1087 u32 c_mdata; 1088 u8 pos, dpos = raid6_gflog[scf]; 1089 dma_addr_t resp_dma = req->tx.phys; 1090 struct brcm_sba_command *cmdsp = cmds; 1091 1092 if (!dst_p) 1093 goto skip_p; 1094 1095 if (pq_continue) { 1096 /* Type-B command to load old P into buf0 */ 1097 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1098 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1099 cmd = sba_cmd_enc(cmd, msg_len, 1100 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1101 c_mdata = sba_cmd_load_c_mdata(0); 1102 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1103 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1104 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 1105 SBA_CMD_SHIFT, SBA_CMD_MASK); 1106 cmdsp->cmd = cmd; 1107 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1108 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1109 cmdsp->data = *dst_p + msg_offset; 1110 cmdsp->data_len = msg_len; 1111 cmdsp++; 1112 1113 /* 1114 * Type-B commands to xor data with buf0 and put it 1115 * back in buf0 1116 */ 1117 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1118 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1119 cmd = sba_cmd_enc(cmd, msg_len, 1120 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1121 c_mdata = sba_cmd_xor_c_mdata(0, 0); 1122 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1123 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1124 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 1125 SBA_CMD_SHIFT, SBA_CMD_MASK); 1126 cmdsp->cmd = cmd; 1127 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1128 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1129 cmdsp->data = src + msg_offset; 1130 cmdsp->data_len = msg_len; 1131 cmdsp++; 1132 } else { 1133 /* Type-B command to load old P into buf0 */ 1134 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1135 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1136 cmd = sba_cmd_enc(cmd, msg_len, 1137 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1138 c_mdata = sba_cmd_load_c_mdata(0); 1139 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1140 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1141 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 1142 SBA_CMD_SHIFT, SBA_CMD_MASK); 1143 cmdsp->cmd = cmd; 1144 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1145 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1146 cmdsp->data = src + msg_offset; 1147 cmdsp->data_len = msg_len; 1148 cmdsp++; 1149 } 1150 1151 /* Type-A command to write buf0 */ 1152 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1153 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1154 cmd = sba_cmd_enc(cmd, msg_len, 1155 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1156 cmd = sba_cmd_enc(cmd, 0x1, 1157 SBA_RESP_SHIFT, SBA_RESP_MASK); 1158 c_mdata = sba_cmd_write_c_mdata(0); 1159 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1160 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1161 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1162 SBA_CMD_SHIFT, SBA_CMD_MASK); 1163 cmdsp->cmd = cmd; 1164 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1165 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1166 if (req->sba->hw_resp_size) { 1167 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1168 cmdsp->resp = resp_dma; 1169 cmdsp->resp_len = req->sba->hw_resp_size; 1170 } 1171 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1172 cmdsp->data = *dst_p + msg_offset; 1173 cmdsp->data_len = msg_len; 1174 cmdsp++; 1175 1176 skip_p: 1177 if (!dst_q) 1178 goto skip_q; 1179 1180 /* Type-A command to zero all buffers */ 1181 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1182 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1183 cmd = sba_cmd_enc(cmd, msg_len, 1184 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1185 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, 1186 SBA_CMD_SHIFT, SBA_CMD_MASK); 1187 cmdsp->cmd = cmd; 1188 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1189 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1190 cmdsp++; 1191 1192 if (dpos == 255) 1193 goto skip_q_computation; 1194 pos = (dpos < req->sba->max_pq_coefs) ? 1195 dpos : (req->sba->max_pq_coefs - 1); 1196 1197 /* 1198 * Type-B command to generate initial Q from data 1199 * and store output into buf0 1200 */ 1201 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1202 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1203 cmd = sba_cmd_enc(cmd, msg_len, 1204 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1205 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0); 1206 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1207 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1208 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1209 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1210 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, 1211 SBA_CMD_SHIFT, SBA_CMD_MASK); 1212 cmdsp->cmd = cmd; 1213 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1214 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1215 cmdsp->data = src + msg_offset; 1216 cmdsp->data_len = msg_len; 1217 cmdsp++; 1218 1219 dpos -= pos; 1220 1221 /* Multiple Type-A command to generate final Q */ 1222 while (dpos) { 1223 pos = (dpos < req->sba->max_pq_coefs) ? 1224 dpos : (req->sba->max_pq_coefs - 1); 1225 1226 /* 1227 * Type-A command to generate Q with buf0 and 1228 * buf1 store result in buf0 1229 */ 1230 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1231 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1232 cmd = sba_cmd_enc(cmd, msg_len, 1233 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1234 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1); 1235 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1236 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1237 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1238 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1239 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, 1240 SBA_CMD_SHIFT, SBA_CMD_MASK); 1241 cmdsp->cmd = cmd; 1242 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1243 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1244 cmdsp++; 1245 1246 dpos -= pos; 1247 } 1248 1249 skip_q_computation: 1250 if (pq_continue) { 1251 /* 1252 * Type-B command to XOR previous output with 1253 * buf0 and write it into buf0 1254 */ 1255 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1256 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1257 cmd = sba_cmd_enc(cmd, msg_len, 1258 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1259 c_mdata = sba_cmd_xor_c_mdata(0, 0); 1260 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1261 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1262 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 1263 SBA_CMD_SHIFT, SBA_CMD_MASK); 1264 cmdsp->cmd = cmd; 1265 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1266 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1267 cmdsp->data = *dst_q + msg_offset; 1268 cmdsp->data_len = msg_len; 1269 cmdsp++; 1270 } 1271 1272 /* Type-A command to write buf0 */ 1273 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1274 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1275 cmd = sba_cmd_enc(cmd, msg_len, 1276 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1277 cmd = sba_cmd_enc(cmd, 0x1, 1278 SBA_RESP_SHIFT, SBA_RESP_MASK); 1279 c_mdata = sba_cmd_write_c_mdata(0); 1280 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1281 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1282 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1283 SBA_CMD_SHIFT, SBA_CMD_MASK); 1284 cmdsp->cmd = cmd; 1285 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1286 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1287 if (req->sba->hw_resp_size) { 1288 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1289 cmdsp->resp = resp_dma; 1290 cmdsp->resp_len = req->sba->hw_resp_size; 1291 } 1292 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1293 cmdsp->data = *dst_q + msg_offset; 1294 cmdsp->data_len = msg_len; 1295 cmdsp++; 1296 1297 skip_q: 1298 /* Fillup brcm_message */ 1299 msg->type = BRCM_MESSAGE_SBA; 1300 msg->sba.cmds = cmds; 1301 msg->sba.cmds_count = cmdsp - cmds; 1302 msg->ctx = req; 1303 msg->error = 0; 1304 } 1305 1306 static struct sba_request * 1307 sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off, 1308 dma_addr_t *dst_p, dma_addr_t *dst_q, 1309 dma_addr_t src, u8 scf, size_t len, 1310 unsigned long flags) 1311 { 1312 struct sba_request *req = NULL; 1313 1314 /* Alloc new request */ 1315 req = sba_alloc_request(sba); 1316 if (!req) 1317 return NULL; 1318 if (flags & DMA_PREP_FENCE) 1319 req->flags |= SBA_REQUEST_FENCE; 1320 1321 /* Fillup request messages */ 1322 sba_fillup_pq_single_msg(req, dmaf_continue(flags), 1323 req->cmds, &req->msg, off, len, 1324 dst_p, dst_q, src, scf); 1325 1326 /* Init async_tx descriptor */ 1327 req->tx.flags = flags; 1328 req->tx.cookie = -EBUSY; 1329 1330 return req; 1331 } 1332 1333 static struct dma_async_tx_descriptor * 1334 sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src, 1335 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) 1336 { 1337 u32 i, dst_q_index; 1338 size_t req_len; 1339 bool slow = false; 1340 dma_addr_t off = 0; 1341 dma_addr_t *dst_p = NULL, *dst_q = NULL; 1342 struct sba_device *sba = to_sba_device(dchan); 1343 struct sba_request *first = NULL, *req; 1344 1345 /* Sanity checks */ 1346 if (unlikely(src_cnt > sba->max_pq_srcs)) 1347 return NULL; 1348 for (i = 0; i < src_cnt; i++) 1349 if (sba->max_pq_coefs <= raid6_gflog[scf[i]]) 1350 slow = true; 1351 1352 /* Figure-out P and Q destination addresses */ 1353 if (!(flags & DMA_PREP_PQ_DISABLE_P)) 1354 dst_p = &dst[0]; 1355 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 1356 dst_q = &dst[1]; 1357 1358 /* Create chained requests where each request is upto hw_buf_size */ 1359 while (len) { 1360 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 1361 1362 if (slow) { 1363 dst_q_index = src_cnt; 1364 1365 if (dst_q) { 1366 for (i = 0; i < src_cnt; i++) { 1367 if (*dst_q == src[i]) { 1368 dst_q_index = i; 1369 break; 1370 } 1371 } 1372 } 1373 1374 if (dst_q_index < src_cnt) { 1375 i = dst_q_index; 1376 req = sba_prep_dma_pq_single_req(sba, 1377 off, dst_p, dst_q, src[i], scf[i], 1378 req_len, flags | DMA_PREP_FENCE); 1379 if (!req) 1380 goto fail; 1381 1382 if (first) 1383 sba_chain_request(first, req); 1384 else 1385 first = req; 1386 1387 flags |= DMA_PREP_CONTINUE; 1388 } 1389 1390 for (i = 0; i < src_cnt; i++) { 1391 if (dst_q_index == i) 1392 continue; 1393 1394 req = sba_prep_dma_pq_single_req(sba, 1395 off, dst_p, dst_q, src[i], scf[i], 1396 req_len, flags | DMA_PREP_FENCE); 1397 if (!req) 1398 goto fail; 1399 1400 if (first) 1401 sba_chain_request(first, req); 1402 else 1403 first = req; 1404 1405 flags |= DMA_PREP_CONTINUE; 1406 } 1407 } else { 1408 req = sba_prep_dma_pq_req(sba, off, 1409 dst_p, dst_q, src, src_cnt, 1410 scf, req_len, flags); 1411 if (!req) 1412 goto fail; 1413 1414 if (first) 1415 sba_chain_request(first, req); 1416 else 1417 first = req; 1418 } 1419 1420 off += req_len; 1421 len -= req_len; 1422 } 1423 1424 return (first) ? &first->tx : NULL; 1425 1426 fail: 1427 if (first) 1428 sba_free_chained_requests(first); 1429 return NULL; 1430 } 1431 1432 /* ====== Mailbox callbacks ===== */ 1433 1434 static void sba_receive_message(struct mbox_client *cl, void *msg) 1435 { 1436 struct brcm_message *m = msg; 1437 struct sba_request *req = m->ctx; 1438 struct sba_device *sba = req->sba; 1439 1440 /* Error count if message has error */ 1441 if (m->error < 0) 1442 dev_err(sba->dev, "%s got message with error %d", 1443 dma_chan_name(&sba->dma_chan), m->error); 1444 1445 /* Process received request */ 1446 sba_process_received_request(sba, req); 1447 } 1448 1449 /* ====== Debugfs callbacks ====== */ 1450 1451 static int sba_debugfs_stats_show(struct seq_file *file, void *offset) 1452 { 1453 struct sba_device *sba = dev_get_drvdata(file->private); 1454 1455 /* Write stats in file */ 1456 sba_write_stats_in_seqfile(sba, file); 1457 1458 return 0; 1459 } 1460 1461 /* ====== Platform driver routines ===== */ 1462 1463 static int sba_prealloc_channel_resources(struct sba_device *sba) 1464 { 1465 int i, j, ret = 0; 1466 struct sba_request *req = NULL; 1467 1468 sba->resp_base = dma_alloc_coherent(sba->mbox_dev, 1469 sba->max_resp_pool_size, 1470 &sba->resp_dma_base, GFP_KERNEL); 1471 if (!sba->resp_base) 1472 return -ENOMEM; 1473 1474 sba->cmds_base = dma_alloc_coherent(sba->mbox_dev, 1475 sba->max_cmds_pool_size, 1476 &sba->cmds_dma_base, GFP_KERNEL); 1477 if (!sba->cmds_base) { 1478 ret = -ENOMEM; 1479 goto fail_free_resp_pool; 1480 } 1481 1482 spin_lock_init(&sba->reqs_lock); 1483 sba->reqs_fence = false; 1484 INIT_LIST_HEAD(&sba->reqs_alloc_list); 1485 INIT_LIST_HEAD(&sba->reqs_pending_list); 1486 INIT_LIST_HEAD(&sba->reqs_active_list); 1487 INIT_LIST_HEAD(&sba->reqs_aborted_list); 1488 INIT_LIST_HEAD(&sba->reqs_free_list); 1489 1490 for (i = 0; i < sba->max_req; i++) { 1491 req = devm_kzalloc(sba->dev, 1492 struct_size(req, cmds, sba->max_cmd_per_req), 1493 GFP_KERNEL); 1494 if (!req) { 1495 ret = -ENOMEM; 1496 goto fail_free_cmds_pool; 1497 } 1498 INIT_LIST_HEAD(&req->node); 1499 req->sba = sba; 1500 req->flags = SBA_REQUEST_STATE_FREE; 1501 INIT_LIST_HEAD(&req->next); 1502 atomic_set(&req->next_pending_count, 0); 1503 for (j = 0; j < sba->max_cmd_per_req; j++) { 1504 req->cmds[j].cmd = 0; 1505 req->cmds[j].cmd_dma = sba->cmds_base + 1506 (i * sba->max_cmd_per_req + j) * sizeof(u64); 1507 req->cmds[j].cmd_dma_addr = sba->cmds_dma_base + 1508 (i * sba->max_cmd_per_req + j) * sizeof(u64); 1509 req->cmds[j].flags = 0; 1510 } 1511 memset(&req->msg, 0, sizeof(req->msg)); 1512 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); 1513 async_tx_ack(&req->tx); 1514 req->tx.tx_submit = sba_tx_submit; 1515 req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size; 1516 list_add_tail(&req->node, &sba->reqs_free_list); 1517 } 1518 1519 return 0; 1520 1521 fail_free_cmds_pool: 1522 dma_free_coherent(sba->mbox_dev, 1523 sba->max_cmds_pool_size, 1524 sba->cmds_base, sba->cmds_dma_base); 1525 fail_free_resp_pool: 1526 dma_free_coherent(sba->mbox_dev, 1527 sba->max_resp_pool_size, 1528 sba->resp_base, sba->resp_dma_base); 1529 return ret; 1530 } 1531 1532 static void sba_freeup_channel_resources(struct sba_device *sba) 1533 { 1534 dmaengine_terminate_all(&sba->dma_chan); 1535 dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size, 1536 sba->cmds_base, sba->cmds_dma_base); 1537 dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size, 1538 sba->resp_base, sba->resp_dma_base); 1539 sba->resp_base = NULL; 1540 sba->resp_dma_base = 0; 1541 } 1542 1543 static int sba_async_register(struct sba_device *sba) 1544 { 1545 int ret; 1546 struct dma_device *dma_dev = &sba->dma_dev; 1547 1548 /* Initialize DMA channel cookie */ 1549 sba->dma_chan.device = dma_dev; 1550 dma_cookie_init(&sba->dma_chan); 1551 1552 /* Initialize DMA device capability mask */ 1553 dma_cap_zero(dma_dev->cap_mask); 1554 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 1555 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 1556 dma_cap_set(DMA_XOR, dma_dev->cap_mask); 1557 dma_cap_set(DMA_PQ, dma_dev->cap_mask); 1558 1559 /* 1560 * Set mailbox channel device as the base device of 1561 * our dma_device because the actual memory accesses 1562 * will be done by mailbox controller 1563 */ 1564 dma_dev->dev = sba->mbox_dev; 1565 1566 /* Set base prep routines */ 1567 dma_dev->device_free_chan_resources = sba_free_chan_resources; 1568 dma_dev->device_terminate_all = sba_device_terminate_all; 1569 dma_dev->device_issue_pending = sba_issue_pending; 1570 dma_dev->device_tx_status = sba_tx_status; 1571 1572 /* Set interrupt routine */ 1573 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1574 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt; 1575 1576 /* Set memcpy routine */ 1577 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1578 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy; 1579 1580 /* Set xor routine and capability */ 1581 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1582 dma_dev->device_prep_dma_xor = sba_prep_dma_xor; 1583 dma_dev->max_xor = sba->max_xor_srcs; 1584 } 1585 1586 /* Set pq routine and capability */ 1587 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 1588 dma_dev->device_prep_dma_pq = sba_prep_dma_pq; 1589 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0); 1590 } 1591 1592 /* Initialize DMA device channel list */ 1593 INIT_LIST_HEAD(&dma_dev->channels); 1594 list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels); 1595 1596 /* Register with Linux async DMA framework*/ 1597 ret = dma_async_device_register(dma_dev); 1598 if (ret) { 1599 dev_err(sba->dev, "async device register error %d", ret); 1600 return ret; 1601 } 1602 1603 dev_info(sba->dev, "%s capabilities: %s%s%s%s\n", 1604 dma_chan_name(&sba->dma_chan), 1605 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "", 1606 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "", 1607 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1608 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : ""); 1609 1610 return 0; 1611 } 1612 1613 static int sba_probe(struct platform_device *pdev) 1614 { 1615 int ret = 0; 1616 struct sba_device *sba; 1617 struct platform_device *mbox_pdev; 1618 struct of_phandle_args args; 1619 1620 /* Allocate main SBA struct */ 1621 sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL); 1622 if (!sba) 1623 return -ENOMEM; 1624 1625 sba->dev = &pdev->dev; 1626 platform_set_drvdata(pdev, sba); 1627 1628 /* Number of mailbox channels should be atleast 1 */ 1629 ret = of_count_phandle_with_args(pdev->dev.of_node, 1630 "mboxes", "#mbox-cells"); 1631 if (ret <= 0) 1632 return -ENODEV; 1633 1634 /* Determine SBA version from DT compatible string */ 1635 if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba")) 1636 sba->ver = SBA_VER_1; 1637 else if (of_device_is_compatible(sba->dev->of_node, 1638 "brcm,iproc-sba-v2")) 1639 sba->ver = SBA_VER_2; 1640 else 1641 return -ENODEV; 1642 1643 /* Derived Configuration parameters */ 1644 switch (sba->ver) { 1645 case SBA_VER_1: 1646 sba->hw_buf_size = 4096; 1647 sba->hw_resp_size = 8; 1648 sba->max_pq_coefs = 6; 1649 sba->max_pq_srcs = 6; 1650 break; 1651 case SBA_VER_2: 1652 sba->hw_buf_size = 4096; 1653 sba->hw_resp_size = 8; 1654 sba->max_pq_coefs = 30; 1655 /* 1656 * We can support max_pq_srcs == max_pq_coefs because 1657 * we are limited by number of SBA commands that we can 1658 * fit in one message for underlying ring manager HW. 1659 */ 1660 sba->max_pq_srcs = 12; 1661 break; 1662 default: 1663 return -EINVAL; 1664 } 1665 sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL; 1666 sba->max_cmd_per_req = sba->max_pq_srcs + 3; 1667 sba->max_xor_srcs = sba->max_cmd_per_req - 1; 1668 sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size; 1669 sba->max_cmds_pool_size = sba->max_req * 1670 sba->max_cmd_per_req * sizeof(u64); 1671 1672 /* Setup mailbox client */ 1673 sba->client.dev = &pdev->dev; 1674 sba->client.rx_callback = sba_receive_message; 1675 sba->client.tx_block = false; 1676 sba->client.knows_txdone = true; 1677 sba->client.tx_tout = 0; 1678 1679 /* Request mailbox channel */ 1680 sba->mchan = mbox_request_channel(&sba->client, 0); 1681 if (IS_ERR(sba->mchan)) { 1682 ret = PTR_ERR(sba->mchan); 1683 goto fail_free_mchan; 1684 } 1685 1686 /* Find-out underlying mailbox device */ 1687 ret = of_parse_phandle_with_args(pdev->dev.of_node, 1688 "mboxes", "#mbox-cells", 0, &args); 1689 if (ret) 1690 goto fail_free_mchan; 1691 mbox_pdev = of_find_device_by_node(args.np); 1692 of_node_put(args.np); 1693 if (!mbox_pdev) { 1694 ret = -ENODEV; 1695 goto fail_free_mchan; 1696 } 1697 sba->mbox_dev = &mbox_pdev->dev; 1698 1699 /* Prealloc channel resource */ 1700 ret = sba_prealloc_channel_resources(sba); 1701 if (ret) 1702 goto fail_free_mchan; 1703 1704 /* Check availability of debugfs */ 1705 if (!debugfs_initialized()) 1706 goto skip_debugfs; 1707 1708 /* Create debugfs root entry */ 1709 sba->root = debugfs_create_dir(dev_name(sba->dev), NULL); 1710 1711 /* Create debugfs stats entry */ 1712 debugfs_create_devm_seqfile(sba->dev, "stats", sba->root, 1713 sba_debugfs_stats_show); 1714 1715 skip_debugfs: 1716 1717 /* Register DMA device with Linux async framework */ 1718 ret = sba_async_register(sba); 1719 if (ret) 1720 goto fail_free_resources; 1721 1722 /* Print device info */ 1723 dev_info(sba->dev, "%s using SBAv%d mailbox channel from %s", 1724 dma_chan_name(&sba->dma_chan), sba->ver+1, 1725 dev_name(sba->mbox_dev)); 1726 1727 return 0; 1728 1729 fail_free_resources: 1730 debugfs_remove_recursive(sba->root); 1731 sba_freeup_channel_resources(sba); 1732 fail_free_mchan: 1733 mbox_free_channel(sba->mchan); 1734 return ret; 1735 } 1736 1737 static int sba_remove(struct platform_device *pdev) 1738 { 1739 struct sba_device *sba = platform_get_drvdata(pdev); 1740 1741 dma_async_device_unregister(&sba->dma_dev); 1742 1743 debugfs_remove_recursive(sba->root); 1744 1745 sba_freeup_channel_resources(sba); 1746 1747 mbox_free_channel(sba->mchan); 1748 1749 return 0; 1750 } 1751 1752 static const struct of_device_id sba_of_match[] = { 1753 { .compatible = "brcm,iproc-sba", }, 1754 { .compatible = "brcm,iproc-sba-v2", }, 1755 {}, 1756 }; 1757 MODULE_DEVICE_TABLE(of, sba_of_match); 1758 1759 static struct platform_driver sba_driver = { 1760 .probe = sba_probe, 1761 .remove = sba_remove, 1762 .driver = { 1763 .name = "bcm-sba-raid", 1764 .of_match_table = sba_of_match, 1765 }, 1766 }; 1767 module_platform_driver(sba_driver); 1768 1769 MODULE_DESCRIPTION("Broadcom SBA RAID driver"); 1770 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>"); 1771 MODULE_LICENSE("GPL v2"); 1772