1 // SPDX-License-Identifier: GPL-2.0-only 2 // Copyright (C) 2017 Broadcom 3 4 /* 5 * Broadcom SBA RAID Driver 6 * 7 * The Broadcom stream buffer accelerator (SBA) provides offloading 8 * capabilities for RAID operations. The SBA offload engine is accessible 9 * via Broadcom SoC specific ring manager. Two or more offload engines 10 * can share same Broadcom SoC specific ring manager due to this Broadcom 11 * SoC specific ring manager driver is implemented as a mailbox controller 12 * driver and offload engine drivers are implemented as mallbox clients. 13 * 14 * Typically, Broadcom SoC specific ring manager will implement larger 15 * number of hardware rings over one or more SBA hardware devices. By 16 * design, the internal buffer size of SBA hardware device is limited 17 * but all offload operations supported by SBA can be broken down into 18 * multiple small size requests and executed parallely on multiple SBA 19 * hardware devices for achieving high through-put. 20 * 21 * The Broadcom SBA RAID driver does not require any register programming 22 * except submitting request to SBA hardware device via mailbox channels. 23 * This driver implements a DMA device with one DMA channel using a single 24 * mailbox channel provided by Broadcom SoC specific ring manager driver. 25 * For having more SBA DMA channels, we can create more SBA device nodes 26 * in Broadcom SoC specific DTS based on number of hardware rings supported 27 * by Broadcom SoC ring manager. 28 */ 29 30 #include <linux/bitops.h> 31 #include <linux/debugfs.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/dmaengine.h> 34 #include <linux/list.h> 35 #include <linux/mailbox_client.h> 36 #include <linux/mailbox/brcm-message.h> 37 #include <linux/module.h> 38 #include <linux/of_device.h> 39 #include <linux/slab.h> 40 #include <linux/raid/pq.h> 41 42 #include "dmaengine.h" 43 44 /* ====== Driver macros and defines ===== */ 45 46 #define SBA_TYPE_SHIFT 48 47 #define SBA_TYPE_MASK GENMASK(1, 0) 48 #define SBA_TYPE_A 0x0 49 #define SBA_TYPE_B 0x2 50 #define SBA_TYPE_C 0x3 51 #define SBA_USER_DEF_SHIFT 32 52 #define SBA_USER_DEF_MASK GENMASK(15, 0) 53 #define SBA_R_MDATA_SHIFT 24 54 #define SBA_R_MDATA_MASK GENMASK(7, 0) 55 #define SBA_C_MDATA_MS_SHIFT 18 56 #define SBA_C_MDATA_MS_MASK GENMASK(1, 0) 57 #define SBA_INT_SHIFT 17 58 #define SBA_INT_MASK BIT(0) 59 #define SBA_RESP_SHIFT 16 60 #define SBA_RESP_MASK BIT(0) 61 #define SBA_C_MDATA_SHIFT 8 62 #define SBA_C_MDATA_MASK GENMASK(7, 0) 63 #define SBA_C_MDATA_BNUMx_SHIFT(__bnum) (2 * (__bnum)) 64 #define SBA_C_MDATA_BNUMx_MASK GENMASK(1, 0) 65 #define SBA_C_MDATA_DNUM_SHIFT 5 66 #define SBA_C_MDATA_DNUM_MASK GENMASK(4, 0) 67 #define SBA_C_MDATA_LS(__v) ((__v) & 0xff) 68 #define SBA_C_MDATA_MS(__v) (((__v) >> 8) & 0x3) 69 #define SBA_CMD_SHIFT 0 70 #define SBA_CMD_MASK GENMASK(3, 0) 71 #define SBA_CMD_ZERO_BUFFER 0x4 72 #define SBA_CMD_ZERO_ALL_BUFFERS 0x8 73 #define SBA_CMD_LOAD_BUFFER 0x9 74 #define SBA_CMD_XOR 0xa 75 #define SBA_CMD_GALOIS_XOR 0xb 76 #define SBA_CMD_WRITE_BUFFER 0xc 77 #define SBA_CMD_GALOIS 0xe 78 79 #define SBA_MAX_REQ_PER_MBOX_CHANNEL 8192 80 #define SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL 8 81 82 /* Driver helper macros */ 83 #define to_sba_request(tx) \ 84 container_of(tx, struct sba_request, tx) 85 #define to_sba_device(dchan) \ 86 container_of(dchan, struct sba_device, dma_chan) 87 88 /* ===== Driver data structures ===== */ 89 90 enum sba_request_flags { 91 SBA_REQUEST_STATE_FREE = 0x001, 92 SBA_REQUEST_STATE_ALLOCED = 0x002, 93 SBA_REQUEST_STATE_PENDING = 0x004, 94 SBA_REQUEST_STATE_ACTIVE = 0x008, 95 SBA_REQUEST_STATE_ABORTED = 0x010, 96 SBA_REQUEST_STATE_MASK = 0x0ff, 97 SBA_REQUEST_FENCE = 0x100, 98 }; 99 100 struct sba_request { 101 /* Global state */ 102 struct list_head node; 103 struct sba_device *sba; 104 u32 flags; 105 /* Chained requests management */ 106 struct sba_request *first; 107 struct list_head next; 108 atomic_t next_pending_count; 109 /* BRCM message data */ 110 struct brcm_message msg; 111 struct dma_async_tx_descriptor tx; 112 /* SBA commands */ 113 struct brcm_sba_command cmds[]; 114 }; 115 116 enum sba_version { 117 SBA_VER_1 = 0, 118 SBA_VER_2 119 }; 120 121 struct sba_device { 122 /* Underlying device */ 123 struct device *dev; 124 /* DT configuration parameters */ 125 enum sba_version ver; 126 /* Derived configuration parameters */ 127 u32 max_req; 128 u32 hw_buf_size; 129 u32 hw_resp_size; 130 u32 max_pq_coefs; 131 u32 max_pq_srcs; 132 u32 max_cmd_per_req; 133 u32 max_xor_srcs; 134 u32 max_resp_pool_size; 135 u32 max_cmds_pool_size; 136 /* Maibox client and Mailbox channels */ 137 struct mbox_client client; 138 struct mbox_chan *mchan; 139 struct device *mbox_dev; 140 /* DMA device and DMA channel */ 141 struct dma_device dma_dev; 142 struct dma_chan dma_chan; 143 /* DMA channel resources */ 144 void *resp_base; 145 dma_addr_t resp_dma_base; 146 void *cmds_base; 147 dma_addr_t cmds_dma_base; 148 spinlock_t reqs_lock; 149 bool reqs_fence; 150 struct list_head reqs_alloc_list; 151 struct list_head reqs_pending_list; 152 struct list_head reqs_active_list; 153 struct list_head reqs_aborted_list; 154 struct list_head reqs_free_list; 155 /* DebugFS directory entries */ 156 struct dentry *root; 157 }; 158 159 /* ====== Command helper routines ===== */ 160 161 static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask) 162 { 163 cmd &= ~((u64)mask << shift); 164 cmd |= ((u64)(val & mask) << shift); 165 return cmd; 166 } 167 168 static inline u32 __pure sba_cmd_load_c_mdata(u32 b0) 169 { 170 return b0 & SBA_C_MDATA_BNUMx_MASK; 171 } 172 173 static inline u32 __pure sba_cmd_write_c_mdata(u32 b0) 174 { 175 return b0 & SBA_C_MDATA_BNUMx_MASK; 176 } 177 178 static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0) 179 { 180 return (b0 & SBA_C_MDATA_BNUMx_MASK) | 181 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)); 182 } 183 184 static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0) 185 { 186 return (b0 & SBA_C_MDATA_BNUMx_MASK) | 187 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) | 188 ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT); 189 } 190 191 /* ====== General helper routines ===== */ 192 193 static struct sba_request *sba_alloc_request(struct sba_device *sba) 194 { 195 bool found = false; 196 unsigned long flags; 197 struct sba_request *req = NULL; 198 199 spin_lock_irqsave(&sba->reqs_lock, flags); 200 list_for_each_entry(req, &sba->reqs_free_list, node) { 201 if (async_tx_test_ack(&req->tx)) { 202 list_move_tail(&req->node, &sba->reqs_alloc_list); 203 found = true; 204 break; 205 } 206 } 207 spin_unlock_irqrestore(&sba->reqs_lock, flags); 208 209 if (!found) { 210 /* 211 * We have no more free requests so, we peek 212 * mailbox channels hoping few active requests 213 * would have completed which will create more 214 * room for new requests. 215 */ 216 mbox_client_peek_data(sba->mchan); 217 return NULL; 218 } 219 220 req->flags = SBA_REQUEST_STATE_ALLOCED; 221 req->first = req; 222 INIT_LIST_HEAD(&req->next); 223 atomic_set(&req->next_pending_count, 1); 224 225 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); 226 async_tx_ack(&req->tx); 227 228 return req; 229 } 230 231 /* Note: Must be called with sba->reqs_lock held */ 232 static void _sba_pending_request(struct sba_device *sba, 233 struct sba_request *req) 234 { 235 lockdep_assert_held(&sba->reqs_lock); 236 req->flags &= ~SBA_REQUEST_STATE_MASK; 237 req->flags |= SBA_REQUEST_STATE_PENDING; 238 list_move_tail(&req->node, &sba->reqs_pending_list); 239 if (list_empty(&sba->reqs_active_list)) 240 sba->reqs_fence = false; 241 } 242 243 /* Note: Must be called with sba->reqs_lock held */ 244 static bool _sba_active_request(struct sba_device *sba, 245 struct sba_request *req) 246 { 247 lockdep_assert_held(&sba->reqs_lock); 248 if (list_empty(&sba->reqs_active_list)) 249 sba->reqs_fence = false; 250 if (sba->reqs_fence) 251 return false; 252 req->flags &= ~SBA_REQUEST_STATE_MASK; 253 req->flags |= SBA_REQUEST_STATE_ACTIVE; 254 list_move_tail(&req->node, &sba->reqs_active_list); 255 if (req->flags & SBA_REQUEST_FENCE) 256 sba->reqs_fence = true; 257 return true; 258 } 259 260 /* Note: Must be called with sba->reqs_lock held */ 261 static void _sba_abort_request(struct sba_device *sba, 262 struct sba_request *req) 263 { 264 lockdep_assert_held(&sba->reqs_lock); 265 req->flags &= ~SBA_REQUEST_STATE_MASK; 266 req->flags |= SBA_REQUEST_STATE_ABORTED; 267 list_move_tail(&req->node, &sba->reqs_aborted_list); 268 if (list_empty(&sba->reqs_active_list)) 269 sba->reqs_fence = false; 270 } 271 272 /* Note: Must be called with sba->reqs_lock held */ 273 static void _sba_free_request(struct sba_device *sba, 274 struct sba_request *req) 275 { 276 lockdep_assert_held(&sba->reqs_lock); 277 req->flags &= ~SBA_REQUEST_STATE_MASK; 278 req->flags |= SBA_REQUEST_STATE_FREE; 279 list_move_tail(&req->node, &sba->reqs_free_list); 280 if (list_empty(&sba->reqs_active_list)) 281 sba->reqs_fence = false; 282 } 283 284 static void sba_free_chained_requests(struct sba_request *req) 285 { 286 unsigned long flags; 287 struct sba_request *nreq; 288 struct sba_device *sba = req->sba; 289 290 spin_lock_irqsave(&sba->reqs_lock, flags); 291 292 _sba_free_request(sba, req); 293 list_for_each_entry(nreq, &req->next, next) 294 _sba_free_request(sba, nreq); 295 296 spin_unlock_irqrestore(&sba->reqs_lock, flags); 297 } 298 299 static void sba_chain_request(struct sba_request *first, 300 struct sba_request *req) 301 { 302 unsigned long flags; 303 struct sba_device *sba = req->sba; 304 305 spin_lock_irqsave(&sba->reqs_lock, flags); 306 307 list_add_tail(&req->next, &first->next); 308 req->first = first; 309 atomic_inc(&first->next_pending_count); 310 311 spin_unlock_irqrestore(&sba->reqs_lock, flags); 312 } 313 314 static void sba_cleanup_nonpending_requests(struct sba_device *sba) 315 { 316 unsigned long flags; 317 struct sba_request *req, *req1; 318 319 spin_lock_irqsave(&sba->reqs_lock, flags); 320 321 /* Freeup all alloced request */ 322 list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node) 323 _sba_free_request(sba, req); 324 325 /* Set all active requests as aborted */ 326 list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node) 327 _sba_abort_request(sba, req); 328 329 /* 330 * Note: We expect that aborted request will be eventually 331 * freed by sba_receive_message() 332 */ 333 334 spin_unlock_irqrestore(&sba->reqs_lock, flags); 335 } 336 337 static void sba_cleanup_pending_requests(struct sba_device *sba) 338 { 339 unsigned long flags; 340 struct sba_request *req, *req1; 341 342 spin_lock_irqsave(&sba->reqs_lock, flags); 343 344 /* Freeup all pending request */ 345 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) 346 _sba_free_request(sba, req); 347 348 spin_unlock_irqrestore(&sba->reqs_lock, flags); 349 } 350 351 static int sba_send_mbox_request(struct sba_device *sba, 352 struct sba_request *req) 353 { 354 int ret = 0; 355 356 /* Send message for the request */ 357 req->msg.error = 0; 358 ret = mbox_send_message(sba->mchan, &req->msg); 359 if (ret < 0) { 360 dev_err(sba->dev, "send message failed with error %d", ret); 361 return ret; 362 } 363 364 /* Check error returned by mailbox controller */ 365 ret = req->msg.error; 366 if (ret < 0) { 367 dev_err(sba->dev, "message error %d", ret); 368 } 369 370 /* Signal txdone for mailbox channel */ 371 mbox_client_txdone(sba->mchan, ret); 372 373 return ret; 374 } 375 376 /* Note: Must be called with sba->reqs_lock held */ 377 static void _sba_process_pending_requests(struct sba_device *sba) 378 { 379 int ret; 380 u32 count; 381 struct sba_request *req; 382 383 /* Process few pending requests */ 384 count = SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL; 385 while (!list_empty(&sba->reqs_pending_list) && count) { 386 /* Get the first pending request */ 387 req = list_first_entry(&sba->reqs_pending_list, 388 struct sba_request, node); 389 390 /* Try to make request active */ 391 if (!_sba_active_request(sba, req)) 392 break; 393 394 /* Send request to mailbox channel */ 395 ret = sba_send_mbox_request(sba, req); 396 if (ret < 0) { 397 _sba_pending_request(sba, req); 398 break; 399 } 400 401 count--; 402 } 403 } 404 405 static void sba_process_received_request(struct sba_device *sba, 406 struct sba_request *req) 407 { 408 unsigned long flags; 409 struct dma_async_tx_descriptor *tx; 410 struct sba_request *nreq, *first = req->first; 411 412 /* Process only after all chained requests are received */ 413 if (!atomic_dec_return(&first->next_pending_count)) { 414 tx = &first->tx; 415 416 WARN_ON(tx->cookie < 0); 417 if (tx->cookie > 0) { 418 spin_lock_irqsave(&sba->reqs_lock, flags); 419 dma_cookie_complete(tx); 420 spin_unlock_irqrestore(&sba->reqs_lock, flags); 421 dmaengine_desc_get_callback_invoke(tx, NULL); 422 dma_descriptor_unmap(tx); 423 tx->callback = NULL; 424 tx->callback_result = NULL; 425 } 426 427 dma_run_dependencies(tx); 428 429 spin_lock_irqsave(&sba->reqs_lock, flags); 430 431 /* Free all requests chained to first request */ 432 list_for_each_entry(nreq, &first->next, next) 433 _sba_free_request(sba, nreq); 434 INIT_LIST_HEAD(&first->next); 435 436 /* Free the first request */ 437 _sba_free_request(sba, first); 438 439 /* Process pending requests */ 440 _sba_process_pending_requests(sba); 441 442 spin_unlock_irqrestore(&sba->reqs_lock, flags); 443 } 444 } 445 446 static void sba_write_stats_in_seqfile(struct sba_device *sba, 447 struct seq_file *file) 448 { 449 unsigned long flags; 450 struct sba_request *req; 451 u32 free_count = 0, alloced_count = 0; 452 u32 pending_count = 0, active_count = 0, aborted_count = 0; 453 454 spin_lock_irqsave(&sba->reqs_lock, flags); 455 456 list_for_each_entry(req, &sba->reqs_free_list, node) 457 if (async_tx_test_ack(&req->tx)) 458 free_count++; 459 460 list_for_each_entry(req, &sba->reqs_alloc_list, node) 461 alloced_count++; 462 463 list_for_each_entry(req, &sba->reqs_pending_list, node) 464 pending_count++; 465 466 list_for_each_entry(req, &sba->reqs_active_list, node) 467 active_count++; 468 469 list_for_each_entry(req, &sba->reqs_aborted_list, node) 470 aborted_count++; 471 472 spin_unlock_irqrestore(&sba->reqs_lock, flags); 473 474 seq_printf(file, "maximum requests = %d\n", sba->max_req); 475 seq_printf(file, "free requests = %d\n", free_count); 476 seq_printf(file, "alloced requests = %d\n", alloced_count); 477 seq_printf(file, "pending requests = %d\n", pending_count); 478 seq_printf(file, "active requests = %d\n", active_count); 479 seq_printf(file, "aborted requests = %d\n", aborted_count); 480 } 481 482 /* ====== DMAENGINE callbacks ===== */ 483 484 static void sba_free_chan_resources(struct dma_chan *dchan) 485 { 486 /* 487 * Channel resources are pre-alloced so we just free-up 488 * whatever we can so that we can re-use pre-alloced 489 * channel resources next time. 490 */ 491 sba_cleanup_nonpending_requests(to_sba_device(dchan)); 492 } 493 494 static int sba_device_terminate_all(struct dma_chan *dchan) 495 { 496 /* Cleanup all pending requests */ 497 sba_cleanup_pending_requests(to_sba_device(dchan)); 498 499 return 0; 500 } 501 502 static void sba_issue_pending(struct dma_chan *dchan) 503 { 504 unsigned long flags; 505 struct sba_device *sba = to_sba_device(dchan); 506 507 /* Process pending requests */ 508 spin_lock_irqsave(&sba->reqs_lock, flags); 509 _sba_process_pending_requests(sba); 510 spin_unlock_irqrestore(&sba->reqs_lock, flags); 511 } 512 513 static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx) 514 { 515 unsigned long flags; 516 dma_cookie_t cookie; 517 struct sba_device *sba; 518 struct sba_request *req, *nreq; 519 520 if (unlikely(!tx)) 521 return -EINVAL; 522 523 sba = to_sba_device(tx->chan); 524 req = to_sba_request(tx); 525 526 /* Assign cookie and mark all chained requests pending */ 527 spin_lock_irqsave(&sba->reqs_lock, flags); 528 cookie = dma_cookie_assign(tx); 529 _sba_pending_request(sba, req); 530 list_for_each_entry(nreq, &req->next, next) 531 _sba_pending_request(sba, nreq); 532 spin_unlock_irqrestore(&sba->reqs_lock, flags); 533 534 return cookie; 535 } 536 537 static enum dma_status sba_tx_status(struct dma_chan *dchan, 538 dma_cookie_t cookie, 539 struct dma_tx_state *txstate) 540 { 541 enum dma_status ret; 542 struct sba_device *sba = to_sba_device(dchan); 543 544 ret = dma_cookie_status(dchan, cookie, txstate); 545 if (ret == DMA_COMPLETE) 546 return ret; 547 548 mbox_client_peek_data(sba->mchan); 549 550 return dma_cookie_status(dchan, cookie, txstate); 551 } 552 553 static void sba_fillup_interrupt_msg(struct sba_request *req, 554 struct brcm_sba_command *cmds, 555 struct brcm_message *msg) 556 { 557 u64 cmd; 558 u32 c_mdata; 559 dma_addr_t resp_dma = req->tx.phys; 560 struct brcm_sba_command *cmdsp = cmds; 561 562 /* Type-B command to load dummy data into buf0 */ 563 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 564 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 565 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size, 566 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 567 c_mdata = sba_cmd_load_c_mdata(0); 568 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 569 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 570 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 571 SBA_CMD_SHIFT, SBA_CMD_MASK); 572 cmdsp->cmd = cmd; 573 *cmdsp->cmd_dma = cpu_to_le64(cmd); 574 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 575 cmdsp->data = resp_dma; 576 cmdsp->data_len = req->sba->hw_resp_size; 577 cmdsp++; 578 579 /* Type-A command to write buf0 to dummy location */ 580 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 581 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 582 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size, 583 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 584 cmd = sba_cmd_enc(cmd, 0x1, 585 SBA_RESP_SHIFT, SBA_RESP_MASK); 586 c_mdata = sba_cmd_write_c_mdata(0); 587 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 588 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 589 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 590 SBA_CMD_SHIFT, SBA_CMD_MASK); 591 cmdsp->cmd = cmd; 592 *cmdsp->cmd_dma = cpu_to_le64(cmd); 593 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 594 if (req->sba->hw_resp_size) { 595 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 596 cmdsp->resp = resp_dma; 597 cmdsp->resp_len = req->sba->hw_resp_size; 598 } 599 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 600 cmdsp->data = resp_dma; 601 cmdsp->data_len = req->sba->hw_resp_size; 602 cmdsp++; 603 604 /* Fillup brcm_message */ 605 msg->type = BRCM_MESSAGE_SBA; 606 msg->sba.cmds = cmds; 607 msg->sba.cmds_count = cmdsp - cmds; 608 msg->ctx = req; 609 msg->error = 0; 610 } 611 612 static struct dma_async_tx_descriptor * 613 sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags) 614 { 615 struct sba_request *req = NULL; 616 struct sba_device *sba = to_sba_device(dchan); 617 618 /* Alloc new request */ 619 req = sba_alloc_request(sba); 620 if (!req) 621 return NULL; 622 623 /* 624 * Force fence so that no requests are submitted 625 * until DMA callback for this request is invoked. 626 */ 627 req->flags |= SBA_REQUEST_FENCE; 628 629 /* Fillup request message */ 630 sba_fillup_interrupt_msg(req, req->cmds, &req->msg); 631 632 /* Init async_tx descriptor */ 633 req->tx.flags = flags; 634 req->tx.cookie = -EBUSY; 635 636 return &req->tx; 637 } 638 639 static void sba_fillup_memcpy_msg(struct sba_request *req, 640 struct brcm_sba_command *cmds, 641 struct brcm_message *msg, 642 dma_addr_t msg_offset, size_t msg_len, 643 dma_addr_t dst, dma_addr_t src) 644 { 645 u64 cmd; 646 u32 c_mdata; 647 dma_addr_t resp_dma = req->tx.phys; 648 struct brcm_sba_command *cmdsp = cmds; 649 650 /* Type-B command to load data into buf0 */ 651 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 652 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 653 cmd = sba_cmd_enc(cmd, msg_len, 654 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 655 c_mdata = sba_cmd_load_c_mdata(0); 656 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 657 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 658 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 659 SBA_CMD_SHIFT, SBA_CMD_MASK); 660 cmdsp->cmd = cmd; 661 *cmdsp->cmd_dma = cpu_to_le64(cmd); 662 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 663 cmdsp->data = src + msg_offset; 664 cmdsp->data_len = msg_len; 665 cmdsp++; 666 667 /* Type-A command to write buf0 */ 668 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 669 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 670 cmd = sba_cmd_enc(cmd, msg_len, 671 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 672 cmd = sba_cmd_enc(cmd, 0x1, 673 SBA_RESP_SHIFT, SBA_RESP_MASK); 674 c_mdata = sba_cmd_write_c_mdata(0); 675 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 676 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 677 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 678 SBA_CMD_SHIFT, SBA_CMD_MASK); 679 cmdsp->cmd = cmd; 680 *cmdsp->cmd_dma = cpu_to_le64(cmd); 681 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 682 if (req->sba->hw_resp_size) { 683 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 684 cmdsp->resp = resp_dma; 685 cmdsp->resp_len = req->sba->hw_resp_size; 686 } 687 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 688 cmdsp->data = dst + msg_offset; 689 cmdsp->data_len = msg_len; 690 cmdsp++; 691 692 /* Fillup brcm_message */ 693 msg->type = BRCM_MESSAGE_SBA; 694 msg->sba.cmds = cmds; 695 msg->sba.cmds_count = cmdsp - cmds; 696 msg->ctx = req; 697 msg->error = 0; 698 } 699 700 static struct sba_request * 701 sba_prep_dma_memcpy_req(struct sba_device *sba, 702 dma_addr_t off, dma_addr_t dst, dma_addr_t src, 703 size_t len, unsigned long flags) 704 { 705 struct sba_request *req = NULL; 706 707 /* Alloc new request */ 708 req = sba_alloc_request(sba); 709 if (!req) 710 return NULL; 711 if (flags & DMA_PREP_FENCE) 712 req->flags |= SBA_REQUEST_FENCE; 713 714 /* Fillup request message */ 715 sba_fillup_memcpy_msg(req, req->cmds, &req->msg, 716 off, len, dst, src); 717 718 /* Init async_tx descriptor */ 719 req->tx.flags = flags; 720 req->tx.cookie = -EBUSY; 721 722 return req; 723 } 724 725 static struct dma_async_tx_descriptor * 726 sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src, 727 size_t len, unsigned long flags) 728 { 729 size_t req_len; 730 dma_addr_t off = 0; 731 struct sba_device *sba = to_sba_device(dchan); 732 struct sba_request *first = NULL, *req; 733 734 /* Create chained requests where each request is upto hw_buf_size */ 735 while (len) { 736 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 737 738 req = sba_prep_dma_memcpy_req(sba, off, dst, src, 739 req_len, flags); 740 if (!req) { 741 if (first) 742 sba_free_chained_requests(first); 743 return NULL; 744 } 745 746 if (first) 747 sba_chain_request(first, req); 748 else 749 first = req; 750 751 off += req_len; 752 len -= req_len; 753 } 754 755 return (first) ? &first->tx : NULL; 756 } 757 758 static void sba_fillup_xor_msg(struct sba_request *req, 759 struct brcm_sba_command *cmds, 760 struct brcm_message *msg, 761 dma_addr_t msg_offset, size_t msg_len, 762 dma_addr_t dst, dma_addr_t *src, u32 src_cnt) 763 { 764 u64 cmd; 765 u32 c_mdata; 766 unsigned int i; 767 dma_addr_t resp_dma = req->tx.phys; 768 struct brcm_sba_command *cmdsp = cmds; 769 770 /* Type-B command to load data into buf0 */ 771 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 772 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 773 cmd = sba_cmd_enc(cmd, msg_len, 774 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 775 c_mdata = sba_cmd_load_c_mdata(0); 776 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 777 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 778 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 779 SBA_CMD_SHIFT, SBA_CMD_MASK); 780 cmdsp->cmd = cmd; 781 *cmdsp->cmd_dma = cpu_to_le64(cmd); 782 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 783 cmdsp->data = src[0] + msg_offset; 784 cmdsp->data_len = msg_len; 785 cmdsp++; 786 787 /* Type-B commands to xor data with buf0 and put it back in buf0 */ 788 for (i = 1; i < src_cnt; i++) { 789 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 790 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 791 cmd = sba_cmd_enc(cmd, msg_len, 792 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 793 c_mdata = sba_cmd_xor_c_mdata(0, 0); 794 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 795 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 796 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 797 SBA_CMD_SHIFT, SBA_CMD_MASK); 798 cmdsp->cmd = cmd; 799 *cmdsp->cmd_dma = cpu_to_le64(cmd); 800 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 801 cmdsp->data = src[i] + msg_offset; 802 cmdsp->data_len = msg_len; 803 cmdsp++; 804 } 805 806 /* Type-A command to write buf0 */ 807 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 808 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 809 cmd = sba_cmd_enc(cmd, msg_len, 810 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 811 cmd = sba_cmd_enc(cmd, 0x1, 812 SBA_RESP_SHIFT, SBA_RESP_MASK); 813 c_mdata = sba_cmd_write_c_mdata(0); 814 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 815 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 816 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 817 SBA_CMD_SHIFT, SBA_CMD_MASK); 818 cmdsp->cmd = cmd; 819 *cmdsp->cmd_dma = cpu_to_le64(cmd); 820 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 821 if (req->sba->hw_resp_size) { 822 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 823 cmdsp->resp = resp_dma; 824 cmdsp->resp_len = req->sba->hw_resp_size; 825 } 826 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 827 cmdsp->data = dst + msg_offset; 828 cmdsp->data_len = msg_len; 829 cmdsp++; 830 831 /* Fillup brcm_message */ 832 msg->type = BRCM_MESSAGE_SBA; 833 msg->sba.cmds = cmds; 834 msg->sba.cmds_count = cmdsp - cmds; 835 msg->ctx = req; 836 msg->error = 0; 837 } 838 839 static struct sba_request * 840 sba_prep_dma_xor_req(struct sba_device *sba, 841 dma_addr_t off, dma_addr_t dst, dma_addr_t *src, 842 u32 src_cnt, size_t len, unsigned long flags) 843 { 844 struct sba_request *req = NULL; 845 846 /* Alloc new request */ 847 req = sba_alloc_request(sba); 848 if (!req) 849 return NULL; 850 if (flags & DMA_PREP_FENCE) 851 req->flags |= SBA_REQUEST_FENCE; 852 853 /* Fillup request message */ 854 sba_fillup_xor_msg(req, req->cmds, &req->msg, 855 off, len, dst, src, src_cnt); 856 857 /* Init async_tx descriptor */ 858 req->tx.flags = flags; 859 req->tx.cookie = -EBUSY; 860 861 return req; 862 } 863 864 static struct dma_async_tx_descriptor * 865 sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src, 866 u32 src_cnt, size_t len, unsigned long flags) 867 { 868 size_t req_len; 869 dma_addr_t off = 0; 870 struct sba_device *sba = to_sba_device(dchan); 871 struct sba_request *first = NULL, *req; 872 873 /* Sanity checks */ 874 if (unlikely(src_cnt > sba->max_xor_srcs)) 875 return NULL; 876 877 /* Create chained requests where each request is upto hw_buf_size */ 878 while (len) { 879 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 880 881 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt, 882 req_len, flags); 883 if (!req) { 884 if (first) 885 sba_free_chained_requests(first); 886 return NULL; 887 } 888 889 if (first) 890 sba_chain_request(first, req); 891 else 892 first = req; 893 894 off += req_len; 895 len -= req_len; 896 } 897 898 return (first) ? &first->tx : NULL; 899 } 900 901 static void sba_fillup_pq_msg(struct sba_request *req, 902 bool pq_continue, 903 struct brcm_sba_command *cmds, 904 struct brcm_message *msg, 905 dma_addr_t msg_offset, size_t msg_len, 906 dma_addr_t *dst_p, dma_addr_t *dst_q, 907 const u8 *scf, dma_addr_t *src, u32 src_cnt) 908 { 909 u64 cmd; 910 u32 c_mdata; 911 unsigned int i; 912 dma_addr_t resp_dma = req->tx.phys; 913 struct brcm_sba_command *cmdsp = cmds; 914 915 if (pq_continue) { 916 /* Type-B command to load old P into buf0 */ 917 if (dst_p) { 918 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 919 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 920 cmd = sba_cmd_enc(cmd, msg_len, 921 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 922 c_mdata = sba_cmd_load_c_mdata(0); 923 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 924 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 925 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 926 SBA_CMD_SHIFT, SBA_CMD_MASK); 927 cmdsp->cmd = cmd; 928 *cmdsp->cmd_dma = cpu_to_le64(cmd); 929 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 930 cmdsp->data = *dst_p + msg_offset; 931 cmdsp->data_len = msg_len; 932 cmdsp++; 933 } 934 935 /* Type-B command to load old Q into buf1 */ 936 if (dst_q) { 937 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 938 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 939 cmd = sba_cmd_enc(cmd, msg_len, 940 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 941 c_mdata = sba_cmd_load_c_mdata(1); 942 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 943 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 944 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 945 SBA_CMD_SHIFT, SBA_CMD_MASK); 946 cmdsp->cmd = cmd; 947 *cmdsp->cmd_dma = cpu_to_le64(cmd); 948 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 949 cmdsp->data = *dst_q + msg_offset; 950 cmdsp->data_len = msg_len; 951 cmdsp++; 952 } 953 } else { 954 /* Type-A command to zero all buffers */ 955 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 956 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 957 cmd = sba_cmd_enc(cmd, msg_len, 958 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 959 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, 960 SBA_CMD_SHIFT, SBA_CMD_MASK); 961 cmdsp->cmd = cmd; 962 *cmdsp->cmd_dma = cpu_to_le64(cmd); 963 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 964 cmdsp++; 965 } 966 967 /* Type-B commands for generate P onto buf0 and Q onto buf1 */ 968 for (i = 0; i < src_cnt; i++) { 969 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 970 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 971 cmd = sba_cmd_enc(cmd, msg_len, 972 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 973 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0); 974 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 975 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 976 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 977 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 978 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR, 979 SBA_CMD_SHIFT, SBA_CMD_MASK); 980 cmdsp->cmd = cmd; 981 *cmdsp->cmd_dma = cpu_to_le64(cmd); 982 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 983 cmdsp->data = src[i] + msg_offset; 984 cmdsp->data_len = msg_len; 985 cmdsp++; 986 } 987 988 /* Type-A command to write buf0 */ 989 if (dst_p) { 990 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 991 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 992 cmd = sba_cmd_enc(cmd, msg_len, 993 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 994 cmd = sba_cmd_enc(cmd, 0x1, 995 SBA_RESP_SHIFT, SBA_RESP_MASK); 996 c_mdata = sba_cmd_write_c_mdata(0); 997 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 998 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 999 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1000 SBA_CMD_SHIFT, SBA_CMD_MASK); 1001 cmdsp->cmd = cmd; 1002 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1003 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1004 if (req->sba->hw_resp_size) { 1005 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1006 cmdsp->resp = resp_dma; 1007 cmdsp->resp_len = req->sba->hw_resp_size; 1008 } 1009 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1010 cmdsp->data = *dst_p + msg_offset; 1011 cmdsp->data_len = msg_len; 1012 cmdsp++; 1013 } 1014 1015 /* Type-A command to write buf1 */ 1016 if (dst_q) { 1017 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1018 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1019 cmd = sba_cmd_enc(cmd, msg_len, 1020 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1021 cmd = sba_cmd_enc(cmd, 0x1, 1022 SBA_RESP_SHIFT, SBA_RESP_MASK); 1023 c_mdata = sba_cmd_write_c_mdata(1); 1024 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1025 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1026 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1027 SBA_CMD_SHIFT, SBA_CMD_MASK); 1028 cmdsp->cmd = cmd; 1029 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1030 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1031 if (req->sba->hw_resp_size) { 1032 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1033 cmdsp->resp = resp_dma; 1034 cmdsp->resp_len = req->sba->hw_resp_size; 1035 } 1036 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1037 cmdsp->data = *dst_q + msg_offset; 1038 cmdsp->data_len = msg_len; 1039 cmdsp++; 1040 } 1041 1042 /* Fillup brcm_message */ 1043 msg->type = BRCM_MESSAGE_SBA; 1044 msg->sba.cmds = cmds; 1045 msg->sba.cmds_count = cmdsp - cmds; 1046 msg->ctx = req; 1047 msg->error = 0; 1048 } 1049 1050 static struct sba_request * 1051 sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off, 1052 dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src, 1053 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) 1054 { 1055 struct sba_request *req = NULL; 1056 1057 /* Alloc new request */ 1058 req = sba_alloc_request(sba); 1059 if (!req) 1060 return NULL; 1061 if (flags & DMA_PREP_FENCE) 1062 req->flags |= SBA_REQUEST_FENCE; 1063 1064 /* Fillup request messages */ 1065 sba_fillup_pq_msg(req, dmaf_continue(flags), 1066 req->cmds, &req->msg, 1067 off, len, dst_p, dst_q, scf, src, src_cnt); 1068 1069 /* Init async_tx descriptor */ 1070 req->tx.flags = flags; 1071 req->tx.cookie = -EBUSY; 1072 1073 return req; 1074 } 1075 1076 static void sba_fillup_pq_single_msg(struct sba_request *req, 1077 bool pq_continue, 1078 struct brcm_sba_command *cmds, 1079 struct brcm_message *msg, 1080 dma_addr_t msg_offset, size_t msg_len, 1081 dma_addr_t *dst_p, dma_addr_t *dst_q, 1082 dma_addr_t src, u8 scf) 1083 { 1084 u64 cmd; 1085 u32 c_mdata; 1086 u8 pos, dpos = raid6_gflog[scf]; 1087 dma_addr_t resp_dma = req->tx.phys; 1088 struct brcm_sba_command *cmdsp = cmds; 1089 1090 if (!dst_p) 1091 goto skip_p; 1092 1093 if (pq_continue) { 1094 /* Type-B command to load old P into buf0 */ 1095 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1096 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1097 cmd = sba_cmd_enc(cmd, msg_len, 1098 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1099 c_mdata = sba_cmd_load_c_mdata(0); 1100 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1101 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1102 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 1103 SBA_CMD_SHIFT, SBA_CMD_MASK); 1104 cmdsp->cmd = cmd; 1105 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1106 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1107 cmdsp->data = *dst_p + msg_offset; 1108 cmdsp->data_len = msg_len; 1109 cmdsp++; 1110 1111 /* 1112 * Type-B commands to xor data with buf0 and put it 1113 * back in buf0 1114 */ 1115 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1116 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1117 cmd = sba_cmd_enc(cmd, msg_len, 1118 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1119 c_mdata = sba_cmd_xor_c_mdata(0, 0); 1120 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1121 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1122 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 1123 SBA_CMD_SHIFT, SBA_CMD_MASK); 1124 cmdsp->cmd = cmd; 1125 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1126 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1127 cmdsp->data = src + msg_offset; 1128 cmdsp->data_len = msg_len; 1129 cmdsp++; 1130 } else { 1131 /* Type-B command to load old P into buf0 */ 1132 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1133 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1134 cmd = sba_cmd_enc(cmd, msg_len, 1135 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1136 c_mdata = sba_cmd_load_c_mdata(0); 1137 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1138 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1139 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 1140 SBA_CMD_SHIFT, SBA_CMD_MASK); 1141 cmdsp->cmd = cmd; 1142 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1143 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1144 cmdsp->data = src + msg_offset; 1145 cmdsp->data_len = msg_len; 1146 cmdsp++; 1147 } 1148 1149 /* Type-A command to write buf0 */ 1150 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1151 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1152 cmd = sba_cmd_enc(cmd, msg_len, 1153 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1154 cmd = sba_cmd_enc(cmd, 0x1, 1155 SBA_RESP_SHIFT, SBA_RESP_MASK); 1156 c_mdata = sba_cmd_write_c_mdata(0); 1157 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1158 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1159 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1160 SBA_CMD_SHIFT, SBA_CMD_MASK); 1161 cmdsp->cmd = cmd; 1162 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1163 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1164 if (req->sba->hw_resp_size) { 1165 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1166 cmdsp->resp = resp_dma; 1167 cmdsp->resp_len = req->sba->hw_resp_size; 1168 } 1169 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1170 cmdsp->data = *dst_p + msg_offset; 1171 cmdsp->data_len = msg_len; 1172 cmdsp++; 1173 1174 skip_p: 1175 if (!dst_q) 1176 goto skip_q; 1177 1178 /* Type-A command to zero all buffers */ 1179 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1180 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1181 cmd = sba_cmd_enc(cmd, msg_len, 1182 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1183 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, 1184 SBA_CMD_SHIFT, SBA_CMD_MASK); 1185 cmdsp->cmd = cmd; 1186 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1187 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1188 cmdsp++; 1189 1190 if (dpos == 255) 1191 goto skip_q_computation; 1192 pos = (dpos < req->sba->max_pq_coefs) ? 1193 dpos : (req->sba->max_pq_coefs - 1); 1194 1195 /* 1196 * Type-B command to generate initial Q from data 1197 * and store output into buf0 1198 */ 1199 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1200 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1201 cmd = sba_cmd_enc(cmd, msg_len, 1202 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1203 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0); 1204 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1205 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1206 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1207 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1208 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, 1209 SBA_CMD_SHIFT, SBA_CMD_MASK); 1210 cmdsp->cmd = cmd; 1211 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1212 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1213 cmdsp->data = src + msg_offset; 1214 cmdsp->data_len = msg_len; 1215 cmdsp++; 1216 1217 dpos -= pos; 1218 1219 /* Multiple Type-A command to generate final Q */ 1220 while (dpos) { 1221 pos = (dpos < req->sba->max_pq_coefs) ? 1222 dpos : (req->sba->max_pq_coefs - 1); 1223 1224 /* 1225 * Type-A command to generate Q with buf0 and 1226 * buf1 store result in buf0 1227 */ 1228 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1229 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1230 cmd = sba_cmd_enc(cmd, msg_len, 1231 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1232 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1); 1233 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1234 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1235 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1236 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1237 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, 1238 SBA_CMD_SHIFT, SBA_CMD_MASK); 1239 cmdsp->cmd = cmd; 1240 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1241 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1242 cmdsp++; 1243 1244 dpos -= pos; 1245 } 1246 1247 skip_q_computation: 1248 if (pq_continue) { 1249 /* 1250 * Type-B command to XOR previous output with 1251 * buf0 and write it into buf0 1252 */ 1253 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1254 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1255 cmd = sba_cmd_enc(cmd, msg_len, 1256 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1257 c_mdata = sba_cmd_xor_c_mdata(0, 0); 1258 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1259 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1260 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 1261 SBA_CMD_SHIFT, SBA_CMD_MASK); 1262 cmdsp->cmd = cmd; 1263 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1264 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1265 cmdsp->data = *dst_q + msg_offset; 1266 cmdsp->data_len = msg_len; 1267 cmdsp++; 1268 } 1269 1270 /* Type-A command to write buf0 */ 1271 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1272 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1273 cmd = sba_cmd_enc(cmd, msg_len, 1274 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1275 cmd = sba_cmd_enc(cmd, 0x1, 1276 SBA_RESP_SHIFT, SBA_RESP_MASK); 1277 c_mdata = sba_cmd_write_c_mdata(0); 1278 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1279 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1280 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1281 SBA_CMD_SHIFT, SBA_CMD_MASK); 1282 cmdsp->cmd = cmd; 1283 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1284 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1285 if (req->sba->hw_resp_size) { 1286 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1287 cmdsp->resp = resp_dma; 1288 cmdsp->resp_len = req->sba->hw_resp_size; 1289 } 1290 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1291 cmdsp->data = *dst_q + msg_offset; 1292 cmdsp->data_len = msg_len; 1293 cmdsp++; 1294 1295 skip_q: 1296 /* Fillup brcm_message */ 1297 msg->type = BRCM_MESSAGE_SBA; 1298 msg->sba.cmds = cmds; 1299 msg->sba.cmds_count = cmdsp - cmds; 1300 msg->ctx = req; 1301 msg->error = 0; 1302 } 1303 1304 static struct sba_request * 1305 sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off, 1306 dma_addr_t *dst_p, dma_addr_t *dst_q, 1307 dma_addr_t src, u8 scf, size_t len, 1308 unsigned long flags) 1309 { 1310 struct sba_request *req = NULL; 1311 1312 /* Alloc new request */ 1313 req = sba_alloc_request(sba); 1314 if (!req) 1315 return NULL; 1316 if (flags & DMA_PREP_FENCE) 1317 req->flags |= SBA_REQUEST_FENCE; 1318 1319 /* Fillup request messages */ 1320 sba_fillup_pq_single_msg(req, dmaf_continue(flags), 1321 req->cmds, &req->msg, off, len, 1322 dst_p, dst_q, src, scf); 1323 1324 /* Init async_tx descriptor */ 1325 req->tx.flags = flags; 1326 req->tx.cookie = -EBUSY; 1327 1328 return req; 1329 } 1330 1331 static struct dma_async_tx_descriptor * 1332 sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src, 1333 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) 1334 { 1335 u32 i, dst_q_index; 1336 size_t req_len; 1337 bool slow = false; 1338 dma_addr_t off = 0; 1339 dma_addr_t *dst_p = NULL, *dst_q = NULL; 1340 struct sba_device *sba = to_sba_device(dchan); 1341 struct sba_request *first = NULL, *req; 1342 1343 /* Sanity checks */ 1344 if (unlikely(src_cnt > sba->max_pq_srcs)) 1345 return NULL; 1346 for (i = 0; i < src_cnt; i++) 1347 if (sba->max_pq_coefs <= raid6_gflog[scf[i]]) 1348 slow = true; 1349 1350 /* Figure-out P and Q destination addresses */ 1351 if (!(flags & DMA_PREP_PQ_DISABLE_P)) 1352 dst_p = &dst[0]; 1353 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 1354 dst_q = &dst[1]; 1355 1356 /* Create chained requests where each request is upto hw_buf_size */ 1357 while (len) { 1358 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 1359 1360 if (slow) { 1361 dst_q_index = src_cnt; 1362 1363 if (dst_q) { 1364 for (i = 0; i < src_cnt; i++) { 1365 if (*dst_q == src[i]) { 1366 dst_q_index = i; 1367 break; 1368 } 1369 } 1370 } 1371 1372 if (dst_q_index < src_cnt) { 1373 i = dst_q_index; 1374 req = sba_prep_dma_pq_single_req(sba, 1375 off, dst_p, dst_q, src[i], scf[i], 1376 req_len, flags | DMA_PREP_FENCE); 1377 if (!req) 1378 goto fail; 1379 1380 if (first) 1381 sba_chain_request(first, req); 1382 else 1383 first = req; 1384 1385 flags |= DMA_PREP_CONTINUE; 1386 } 1387 1388 for (i = 0; i < src_cnt; i++) { 1389 if (dst_q_index == i) 1390 continue; 1391 1392 req = sba_prep_dma_pq_single_req(sba, 1393 off, dst_p, dst_q, src[i], scf[i], 1394 req_len, flags | DMA_PREP_FENCE); 1395 if (!req) 1396 goto fail; 1397 1398 if (first) 1399 sba_chain_request(first, req); 1400 else 1401 first = req; 1402 1403 flags |= DMA_PREP_CONTINUE; 1404 } 1405 } else { 1406 req = sba_prep_dma_pq_req(sba, off, 1407 dst_p, dst_q, src, src_cnt, 1408 scf, req_len, flags); 1409 if (!req) 1410 goto fail; 1411 1412 if (first) 1413 sba_chain_request(first, req); 1414 else 1415 first = req; 1416 } 1417 1418 off += req_len; 1419 len -= req_len; 1420 } 1421 1422 return (first) ? &first->tx : NULL; 1423 1424 fail: 1425 if (first) 1426 sba_free_chained_requests(first); 1427 return NULL; 1428 } 1429 1430 /* ====== Mailbox callbacks ===== */ 1431 1432 static void sba_receive_message(struct mbox_client *cl, void *msg) 1433 { 1434 struct brcm_message *m = msg; 1435 struct sba_request *req = m->ctx; 1436 struct sba_device *sba = req->sba; 1437 1438 /* Error count if message has error */ 1439 if (m->error < 0) 1440 dev_err(sba->dev, "%s got message with error %d", 1441 dma_chan_name(&sba->dma_chan), m->error); 1442 1443 /* Process received request */ 1444 sba_process_received_request(sba, req); 1445 } 1446 1447 /* ====== Debugfs callbacks ====== */ 1448 1449 static int sba_debugfs_stats_show(struct seq_file *file, void *offset) 1450 { 1451 struct sba_device *sba = dev_get_drvdata(file->private); 1452 1453 /* Write stats in file */ 1454 sba_write_stats_in_seqfile(sba, file); 1455 1456 return 0; 1457 } 1458 1459 /* ====== Platform driver routines ===== */ 1460 1461 static int sba_prealloc_channel_resources(struct sba_device *sba) 1462 { 1463 int i, j, ret = 0; 1464 struct sba_request *req = NULL; 1465 1466 sba->resp_base = dma_alloc_coherent(sba->mbox_dev, 1467 sba->max_resp_pool_size, 1468 &sba->resp_dma_base, GFP_KERNEL); 1469 if (!sba->resp_base) 1470 return -ENOMEM; 1471 1472 sba->cmds_base = dma_alloc_coherent(sba->mbox_dev, 1473 sba->max_cmds_pool_size, 1474 &sba->cmds_dma_base, GFP_KERNEL); 1475 if (!sba->cmds_base) { 1476 ret = -ENOMEM; 1477 goto fail_free_resp_pool; 1478 } 1479 1480 spin_lock_init(&sba->reqs_lock); 1481 sba->reqs_fence = false; 1482 INIT_LIST_HEAD(&sba->reqs_alloc_list); 1483 INIT_LIST_HEAD(&sba->reqs_pending_list); 1484 INIT_LIST_HEAD(&sba->reqs_active_list); 1485 INIT_LIST_HEAD(&sba->reqs_aborted_list); 1486 INIT_LIST_HEAD(&sba->reqs_free_list); 1487 1488 for (i = 0; i < sba->max_req; i++) { 1489 req = devm_kzalloc(sba->dev, 1490 struct_size(req, cmds, sba->max_cmd_per_req), 1491 GFP_KERNEL); 1492 if (!req) { 1493 ret = -ENOMEM; 1494 goto fail_free_cmds_pool; 1495 } 1496 INIT_LIST_HEAD(&req->node); 1497 req->sba = sba; 1498 req->flags = SBA_REQUEST_STATE_FREE; 1499 INIT_LIST_HEAD(&req->next); 1500 atomic_set(&req->next_pending_count, 0); 1501 for (j = 0; j < sba->max_cmd_per_req; j++) { 1502 req->cmds[j].cmd = 0; 1503 req->cmds[j].cmd_dma = sba->cmds_base + 1504 (i * sba->max_cmd_per_req + j) * sizeof(u64); 1505 req->cmds[j].cmd_dma_addr = sba->cmds_dma_base + 1506 (i * sba->max_cmd_per_req + j) * sizeof(u64); 1507 req->cmds[j].flags = 0; 1508 } 1509 memset(&req->msg, 0, sizeof(req->msg)); 1510 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); 1511 async_tx_ack(&req->tx); 1512 req->tx.tx_submit = sba_tx_submit; 1513 req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size; 1514 list_add_tail(&req->node, &sba->reqs_free_list); 1515 } 1516 1517 return 0; 1518 1519 fail_free_cmds_pool: 1520 dma_free_coherent(sba->mbox_dev, 1521 sba->max_cmds_pool_size, 1522 sba->cmds_base, sba->cmds_dma_base); 1523 fail_free_resp_pool: 1524 dma_free_coherent(sba->mbox_dev, 1525 sba->max_resp_pool_size, 1526 sba->resp_base, sba->resp_dma_base); 1527 return ret; 1528 } 1529 1530 static void sba_freeup_channel_resources(struct sba_device *sba) 1531 { 1532 dmaengine_terminate_all(&sba->dma_chan); 1533 dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size, 1534 sba->cmds_base, sba->cmds_dma_base); 1535 dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size, 1536 sba->resp_base, sba->resp_dma_base); 1537 sba->resp_base = NULL; 1538 sba->resp_dma_base = 0; 1539 } 1540 1541 static int sba_async_register(struct sba_device *sba) 1542 { 1543 int ret; 1544 struct dma_device *dma_dev = &sba->dma_dev; 1545 1546 /* Initialize DMA channel cookie */ 1547 sba->dma_chan.device = dma_dev; 1548 dma_cookie_init(&sba->dma_chan); 1549 1550 /* Initialize DMA device capability mask */ 1551 dma_cap_zero(dma_dev->cap_mask); 1552 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 1553 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 1554 dma_cap_set(DMA_XOR, dma_dev->cap_mask); 1555 dma_cap_set(DMA_PQ, dma_dev->cap_mask); 1556 1557 /* 1558 * Set mailbox channel device as the base device of 1559 * our dma_device because the actual memory accesses 1560 * will be done by mailbox controller 1561 */ 1562 dma_dev->dev = sba->mbox_dev; 1563 1564 /* Set base prep routines */ 1565 dma_dev->device_free_chan_resources = sba_free_chan_resources; 1566 dma_dev->device_terminate_all = sba_device_terminate_all; 1567 dma_dev->device_issue_pending = sba_issue_pending; 1568 dma_dev->device_tx_status = sba_tx_status; 1569 1570 /* Set interrupt routine */ 1571 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1572 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt; 1573 1574 /* Set memcpy routine */ 1575 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1576 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy; 1577 1578 /* Set xor routine and capability */ 1579 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1580 dma_dev->device_prep_dma_xor = sba_prep_dma_xor; 1581 dma_dev->max_xor = sba->max_xor_srcs; 1582 } 1583 1584 /* Set pq routine and capability */ 1585 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 1586 dma_dev->device_prep_dma_pq = sba_prep_dma_pq; 1587 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0); 1588 } 1589 1590 /* Initialize DMA device channel list */ 1591 INIT_LIST_HEAD(&dma_dev->channels); 1592 list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels); 1593 1594 /* Register with Linux async DMA framework*/ 1595 ret = dma_async_device_register(dma_dev); 1596 if (ret) { 1597 dev_err(sba->dev, "async device register error %d", ret); 1598 return ret; 1599 } 1600 1601 dev_info(sba->dev, "%s capabilities: %s%s%s%s\n", 1602 dma_chan_name(&sba->dma_chan), 1603 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "", 1604 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "", 1605 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1606 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : ""); 1607 1608 return 0; 1609 } 1610 1611 static int sba_probe(struct platform_device *pdev) 1612 { 1613 int ret = 0; 1614 struct sba_device *sba; 1615 struct platform_device *mbox_pdev; 1616 struct of_phandle_args args; 1617 1618 /* Allocate main SBA struct */ 1619 sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL); 1620 if (!sba) 1621 return -ENOMEM; 1622 1623 sba->dev = &pdev->dev; 1624 platform_set_drvdata(pdev, sba); 1625 1626 /* Number of mailbox channels should be atleast 1 */ 1627 ret = of_count_phandle_with_args(pdev->dev.of_node, 1628 "mboxes", "#mbox-cells"); 1629 if (ret <= 0) 1630 return -ENODEV; 1631 1632 /* Determine SBA version from DT compatible string */ 1633 if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba")) 1634 sba->ver = SBA_VER_1; 1635 else if (of_device_is_compatible(sba->dev->of_node, 1636 "brcm,iproc-sba-v2")) 1637 sba->ver = SBA_VER_2; 1638 else 1639 return -ENODEV; 1640 1641 /* Derived Configuration parameters */ 1642 switch (sba->ver) { 1643 case SBA_VER_1: 1644 sba->hw_buf_size = 4096; 1645 sba->hw_resp_size = 8; 1646 sba->max_pq_coefs = 6; 1647 sba->max_pq_srcs = 6; 1648 break; 1649 case SBA_VER_2: 1650 sba->hw_buf_size = 4096; 1651 sba->hw_resp_size = 8; 1652 sba->max_pq_coefs = 30; 1653 /* 1654 * We can support max_pq_srcs == max_pq_coefs because 1655 * we are limited by number of SBA commands that we can 1656 * fit in one message for underlying ring manager HW. 1657 */ 1658 sba->max_pq_srcs = 12; 1659 break; 1660 default: 1661 return -EINVAL; 1662 } 1663 sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL; 1664 sba->max_cmd_per_req = sba->max_pq_srcs + 3; 1665 sba->max_xor_srcs = sba->max_cmd_per_req - 1; 1666 sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size; 1667 sba->max_cmds_pool_size = sba->max_req * 1668 sba->max_cmd_per_req * sizeof(u64); 1669 1670 /* Setup mailbox client */ 1671 sba->client.dev = &pdev->dev; 1672 sba->client.rx_callback = sba_receive_message; 1673 sba->client.tx_block = false; 1674 sba->client.knows_txdone = true; 1675 sba->client.tx_tout = 0; 1676 1677 /* Request mailbox channel */ 1678 sba->mchan = mbox_request_channel(&sba->client, 0); 1679 if (IS_ERR(sba->mchan)) { 1680 ret = PTR_ERR(sba->mchan); 1681 goto fail_free_mchan; 1682 } 1683 1684 /* Find-out underlying mailbox device */ 1685 ret = of_parse_phandle_with_args(pdev->dev.of_node, 1686 "mboxes", "#mbox-cells", 0, &args); 1687 if (ret) 1688 goto fail_free_mchan; 1689 mbox_pdev = of_find_device_by_node(args.np); 1690 of_node_put(args.np); 1691 if (!mbox_pdev) { 1692 ret = -ENODEV; 1693 goto fail_free_mchan; 1694 } 1695 sba->mbox_dev = &mbox_pdev->dev; 1696 1697 /* Prealloc channel resource */ 1698 ret = sba_prealloc_channel_resources(sba); 1699 if (ret) 1700 goto fail_free_mchan; 1701 1702 /* Check availability of debugfs */ 1703 if (!debugfs_initialized()) 1704 goto skip_debugfs; 1705 1706 /* Create debugfs root entry */ 1707 sba->root = debugfs_create_dir(dev_name(sba->dev), NULL); 1708 1709 /* Create debugfs stats entry */ 1710 debugfs_create_devm_seqfile(sba->dev, "stats", sba->root, 1711 sba_debugfs_stats_show); 1712 1713 skip_debugfs: 1714 1715 /* Register DMA device with Linux async framework */ 1716 ret = sba_async_register(sba); 1717 if (ret) 1718 goto fail_free_resources; 1719 1720 /* Print device info */ 1721 dev_info(sba->dev, "%s using SBAv%d mailbox channel from %s", 1722 dma_chan_name(&sba->dma_chan), sba->ver+1, 1723 dev_name(sba->mbox_dev)); 1724 1725 return 0; 1726 1727 fail_free_resources: 1728 debugfs_remove_recursive(sba->root); 1729 sba_freeup_channel_resources(sba); 1730 fail_free_mchan: 1731 mbox_free_channel(sba->mchan); 1732 return ret; 1733 } 1734 1735 static int sba_remove(struct platform_device *pdev) 1736 { 1737 struct sba_device *sba = platform_get_drvdata(pdev); 1738 1739 dma_async_device_unregister(&sba->dma_dev); 1740 1741 debugfs_remove_recursive(sba->root); 1742 1743 sba_freeup_channel_resources(sba); 1744 1745 mbox_free_channel(sba->mchan); 1746 1747 return 0; 1748 } 1749 1750 static const struct of_device_id sba_of_match[] = { 1751 { .compatible = "brcm,iproc-sba", }, 1752 { .compatible = "brcm,iproc-sba-v2", }, 1753 {}, 1754 }; 1755 MODULE_DEVICE_TABLE(of, sba_of_match); 1756 1757 static struct platform_driver sba_driver = { 1758 .probe = sba_probe, 1759 .remove = sba_remove, 1760 .driver = { 1761 .name = "bcm-sba-raid", 1762 .of_match_table = sba_of_match, 1763 }, 1764 }; 1765 module_platform_driver(sba_driver); 1766 1767 MODULE_DESCRIPTION("Broadcom SBA RAID driver"); 1768 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>"); 1769 MODULE_LICENSE("GPL v2"); 1770