1 /* 2 * Copyright (C) 2017 Broadcom 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 /* 10 * Broadcom SBA RAID Driver 11 * 12 * The Broadcom stream buffer accelerator (SBA) provides offloading 13 * capabilities for RAID operations. The SBA offload engine is accessible 14 * via Broadcom SoC specific ring manager. Two or more offload engines 15 * can share same Broadcom SoC specific ring manager due to this Broadcom 16 * SoC specific ring manager driver is implemented as a mailbox controller 17 * driver and offload engine drivers are implemented as mallbox clients. 18 * 19 * Typically, Broadcom SoC specific ring manager will implement larger 20 * number of hardware rings over one or more SBA hardware devices. By 21 * design, the internal buffer size of SBA hardware device is limited 22 * but all offload operations supported by SBA can be broken down into 23 * multiple small size requests and executed parallely on multiple SBA 24 * hardware devices for achieving high through-put. 25 * 26 * The Broadcom SBA RAID driver does not require any register programming 27 * except submitting request to SBA hardware device via mailbox channels. 28 * This driver implements a DMA device with one DMA channel using a set 29 * of mailbox channels provided by Broadcom SoC specific ring manager 30 * driver. To exploit parallelism (as described above), all DMA request 31 * coming to SBA RAID DMA channel are broken down to smaller requests 32 * and submitted to multiple mailbox channels in round-robin fashion. 33 * For having more SBA DMA channels, we can create more SBA device nodes 34 * in Broadcom SoC specific DTS based on number of hardware rings supported 35 * by Broadcom SoC ring manager. 36 */ 37 38 #include <linux/bitops.h> 39 #include <linux/debugfs.h> 40 #include <linux/dma-mapping.h> 41 #include <linux/dmaengine.h> 42 #include <linux/list.h> 43 #include <linux/mailbox_client.h> 44 #include <linux/mailbox/brcm-message.h> 45 #include <linux/module.h> 46 #include <linux/of_device.h> 47 #include <linux/slab.h> 48 #include <linux/raid/pq.h> 49 50 #include "dmaengine.h" 51 52 /* ====== Driver macros and defines ===== */ 53 54 #define SBA_TYPE_SHIFT 48 55 #define SBA_TYPE_MASK GENMASK(1, 0) 56 #define SBA_TYPE_A 0x0 57 #define SBA_TYPE_B 0x2 58 #define SBA_TYPE_C 0x3 59 #define SBA_USER_DEF_SHIFT 32 60 #define SBA_USER_DEF_MASK GENMASK(15, 0) 61 #define SBA_R_MDATA_SHIFT 24 62 #define SBA_R_MDATA_MASK GENMASK(7, 0) 63 #define SBA_C_MDATA_MS_SHIFT 18 64 #define SBA_C_MDATA_MS_MASK GENMASK(1, 0) 65 #define SBA_INT_SHIFT 17 66 #define SBA_INT_MASK BIT(0) 67 #define SBA_RESP_SHIFT 16 68 #define SBA_RESP_MASK BIT(0) 69 #define SBA_C_MDATA_SHIFT 8 70 #define SBA_C_MDATA_MASK GENMASK(7, 0) 71 #define SBA_C_MDATA_BNUMx_SHIFT(__bnum) (2 * (__bnum)) 72 #define SBA_C_MDATA_BNUMx_MASK GENMASK(1, 0) 73 #define SBA_C_MDATA_DNUM_SHIFT 5 74 #define SBA_C_MDATA_DNUM_MASK GENMASK(4, 0) 75 #define SBA_C_MDATA_LS(__v) ((__v) & 0xff) 76 #define SBA_C_MDATA_MS(__v) (((__v) >> 8) & 0x3) 77 #define SBA_CMD_SHIFT 0 78 #define SBA_CMD_MASK GENMASK(3, 0) 79 #define SBA_CMD_ZERO_BUFFER 0x4 80 #define SBA_CMD_ZERO_ALL_BUFFERS 0x8 81 #define SBA_CMD_LOAD_BUFFER 0x9 82 #define SBA_CMD_XOR 0xa 83 #define SBA_CMD_GALOIS_XOR 0xb 84 #define SBA_CMD_WRITE_BUFFER 0xc 85 #define SBA_CMD_GALOIS 0xe 86 87 #define SBA_MAX_REQ_PER_MBOX_CHANNEL 8192 88 89 /* Driver helper macros */ 90 #define to_sba_request(tx) \ 91 container_of(tx, struct sba_request, tx) 92 #define to_sba_device(dchan) \ 93 container_of(dchan, struct sba_device, dma_chan) 94 95 /* ===== Driver data structures ===== */ 96 97 enum sba_request_flags { 98 SBA_REQUEST_STATE_FREE = 0x001, 99 SBA_REQUEST_STATE_ALLOCED = 0x002, 100 SBA_REQUEST_STATE_PENDING = 0x004, 101 SBA_REQUEST_STATE_ACTIVE = 0x008, 102 SBA_REQUEST_STATE_ABORTED = 0x010, 103 SBA_REQUEST_STATE_MASK = 0x0ff, 104 SBA_REQUEST_FENCE = 0x100, 105 }; 106 107 struct sba_request { 108 /* Global state */ 109 struct list_head node; 110 struct sba_device *sba; 111 u32 flags; 112 /* Chained requests management */ 113 struct sba_request *first; 114 struct list_head next; 115 atomic_t next_pending_count; 116 /* BRCM message data */ 117 struct brcm_message msg; 118 struct dma_async_tx_descriptor tx; 119 /* SBA commands */ 120 struct brcm_sba_command cmds[0]; 121 }; 122 123 enum sba_version { 124 SBA_VER_1 = 0, 125 SBA_VER_2 126 }; 127 128 struct sba_device { 129 /* Underlying device */ 130 struct device *dev; 131 /* DT configuration parameters */ 132 enum sba_version ver; 133 /* Derived configuration parameters */ 134 u32 max_req; 135 u32 hw_buf_size; 136 u32 hw_resp_size; 137 u32 max_pq_coefs; 138 u32 max_pq_srcs; 139 u32 max_cmd_per_req; 140 u32 max_xor_srcs; 141 u32 max_resp_pool_size; 142 u32 max_cmds_pool_size; 143 /* Maibox client and Mailbox channels */ 144 struct mbox_client client; 145 int mchans_count; 146 atomic_t mchans_current; 147 struct mbox_chan **mchans; 148 struct device *mbox_dev; 149 /* DMA device and DMA channel */ 150 struct dma_device dma_dev; 151 struct dma_chan dma_chan; 152 /* DMA channel resources */ 153 void *resp_base; 154 dma_addr_t resp_dma_base; 155 void *cmds_base; 156 dma_addr_t cmds_dma_base; 157 spinlock_t reqs_lock; 158 bool reqs_fence; 159 struct list_head reqs_alloc_list; 160 struct list_head reqs_pending_list; 161 struct list_head reqs_active_list; 162 struct list_head reqs_aborted_list; 163 struct list_head reqs_free_list; 164 /* DebugFS directory entries */ 165 struct dentry *root; 166 struct dentry *stats; 167 }; 168 169 /* ====== Command helper routines ===== */ 170 171 static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask) 172 { 173 cmd &= ~((u64)mask << shift); 174 cmd |= ((u64)(val & mask) << shift); 175 return cmd; 176 } 177 178 static inline u32 __pure sba_cmd_load_c_mdata(u32 b0) 179 { 180 return b0 & SBA_C_MDATA_BNUMx_MASK; 181 } 182 183 static inline u32 __pure sba_cmd_write_c_mdata(u32 b0) 184 { 185 return b0 & SBA_C_MDATA_BNUMx_MASK; 186 } 187 188 static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0) 189 { 190 return (b0 & SBA_C_MDATA_BNUMx_MASK) | 191 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)); 192 } 193 194 static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0) 195 { 196 return (b0 & SBA_C_MDATA_BNUMx_MASK) | 197 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) | 198 ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT); 199 } 200 201 /* ====== General helper routines ===== */ 202 203 static void sba_peek_mchans(struct sba_device *sba) 204 { 205 int mchan_idx; 206 207 for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++) 208 mbox_client_peek_data(sba->mchans[mchan_idx]); 209 } 210 211 static struct sba_request *sba_alloc_request(struct sba_device *sba) 212 { 213 bool found = false; 214 unsigned long flags; 215 struct sba_request *req = NULL; 216 217 spin_lock_irqsave(&sba->reqs_lock, flags); 218 list_for_each_entry(req, &sba->reqs_free_list, node) { 219 if (async_tx_test_ack(&req->tx)) { 220 list_move_tail(&req->node, &sba->reqs_alloc_list); 221 found = true; 222 break; 223 } 224 } 225 spin_unlock_irqrestore(&sba->reqs_lock, flags); 226 227 if (!found) { 228 /* 229 * We have no more free requests so, we peek 230 * mailbox channels hoping few active requests 231 * would have completed which will create more 232 * room for new requests. 233 */ 234 sba_peek_mchans(sba); 235 return NULL; 236 } 237 238 req->flags = SBA_REQUEST_STATE_ALLOCED; 239 req->first = req; 240 INIT_LIST_HEAD(&req->next); 241 atomic_set(&req->next_pending_count, 1); 242 243 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); 244 async_tx_ack(&req->tx); 245 246 return req; 247 } 248 249 /* Note: Must be called with sba->reqs_lock held */ 250 static void _sba_pending_request(struct sba_device *sba, 251 struct sba_request *req) 252 { 253 lockdep_assert_held(&sba->reqs_lock); 254 req->flags &= ~SBA_REQUEST_STATE_MASK; 255 req->flags |= SBA_REQUEST_STATE_PENDING; 256 list_move_tail(&req->node, &sba->reqs_pending_list); 257 if (list_empty(&sba->reqs_active_list)) 258 sba->reqs_fence = false; 259 } 260 261 /* Note: Must be called with sba->reqs_lock held */ 262 static bool _sba_active_request(struct sba_device *sba, 263 struct sba_request *req) 264 { 265 lockdep_assert_held(&sba->reqs_lock); 266 if (list_empty(&sba->reqs_active_list)) 267 sba->reqs_fence = false; 268 if (sba->reqs_fence) 269 return false; 270 req->flags &= ~SBA_REQUEST_STATE_MASK; 271 req->flags |= SBA_REQUEST_STATE_ACTIVE; 272 list_move_tail(&req->node, &sba->reqs_active_list); 273 if (req->flags & SBA_REQUEST_FENCE) 274 sba->reqs_fence = true; 275 return true; 276 } 277 278 /* Note: Must be called with sba->reqs_lock held */ 279 static void _sba_abort_request(struct sba_device *sba, 280 struct sba_request *req) 281 { 282 lockdep_assert_held(&sba->reqs_lock); 283 req->flags &= ~SBA_REQUEST_STATE_MASK; 284 req->flags |= SBA_REQUEST_STATE_ABORTED; 285 list_move_tail(&req->node, &sba->reqs_aborted_list); 286 if (list_empty(&sba->reqs_active_list)) 287 sba->reqs_fence = false; 288 } 289 290 /* Note: Must be called with sba->reqs_lock held */ 291 static void _sba_free_request(struct sba_device *sba, 292 struct sba_request *req) 293 { 294 lockdep_assert_held(&sba->reqs_lock); 295 req->flags &= ~SBA_REQUEST_STATE_MASK; 296 req->flags |= SBA_REQUEST_STATE_FREE; 297 list_move_tail(&req->node, &sba->reqs_free_list); 298 if (list_empty(&sba->reqs_active_list)) 299 sba->reqs_fence = false; 300 } 301 302 static void sba_free_chained_requests(struct sba_request *req) 303 { 304 unsigned long flags; 305 struct sba_request *nreq; 306 struct sba_device *sba = req->sba; 307 308 spin_lock_irqsave(&sba->reqs_lock, flags); 309 310 _sba_free_request(sba, req); 311 list_for_each_entry(nreq, &req->next, next) 312 _sba_free_request(sba, nreq); 313 314 spin_unlock_irqrestore(&sba->reqs_lock, flags); 315 } 316 317 static void sba_chain_request(struct sba_request *first, 318 struct sba_request *req) 319 { 320 unsigned long flags; 321 struct sba_device *sba = req->sba; 322 323 spin_lock_irqsave(&sba->reqs_lock, flags); 324 325 list_add_tail(&req->next, &first->next); 326 req->first = first; 327 atomic_inc(&first->next_pending_count); 328 329 spin_unlock_irqrestore(&sba->reqs_lock, flags); 330 } 331 332 static void sba_cleanup_nonpending_requests(struct sba_device *sba) 333 { 334 unsigned long flags; 335 struct sba_request *req, *req1; 336 337 spin_lock_irqsave(&sba->reqs_lock, flags); 338 339 /* Freeup all alloced request */ 340 list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node) 341 _sba_free_request(sba, req); 342 343 /* Set all active requests as aborted */ 344 list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node) 345 _sba_abort_request(sba, req); 346 347 /* 348 * Note: We expect that aborted request will be eventually 349 * freed by sba_receive_message() 350 */ 351 352 spin_unlock_irqrestore(&sba->reqs_lock, flags); 353 } 354 355 static void sba_cleanup_pending_requests(struct sba_device *sba) 356 { 357 unsigned long flags; 358 struct sba_request *req, *req1; 359 360 spin_lock_irqsave(&sba->reqs_lock, flags); 361 362 /* Freeup all pending request */ 363 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) 364 _sba_free_request(sba, req); 365 366 spin_unlock_irqrestore(&sba->reqs_lock, flags); 367 } 368 369 static int sba_send_mbox_request(struct sba_device *sba, 370 struct sba_request *req) 371 { 372 int mchans_idx, ret = 0; 373 374 /* Select mailbox channel in round-robin fashion */ 375 mchans_idx = atomic_inc_return(&sba->mchans_current); 376 mchans_idx = mchans_idx % sba->mchans_count; 377 378 /* Send message for the request */ 379 req->msg.error = 0; 380 ret = mbox_send_message(sba->mchans[mchans_idx], &req->msg); 381 if (ret < 0) { 382 dev_err(sba->dev, "send message failed with error %d", ret); 383 return ret; 384 } 385 386 /* Check error returned by mailbox controller */ 387 ret = req->msg.error; 388 if (ret < 0) { 389 dev_err(sba->dev, "message error %d", ret); 390 } 391 392 /* Signal txdone for mailbox channel */ 393 mbox_client_txdone(sba->mchans[mchans_idx], ret); 394 395 return ret; 396 } 397 398 /* Note: Must be called with sba->reqs_lock held */ 399 static void _sba_process_pending_requests(struct sba_device *sba) 400 { 401 int ret; 402 u32 count; 403 struct sba_request *req; 404 405 /* 406 * Process few pending requests 407 * 408 * For now, we process (<number_of_mailbox_channels> * 8) 409 * number of requests at a time. 410 */ 411 count = sba->mchans_count * 8; 412 while (!list_empty(&sba->reqs_pending_list) && count) { 413 /* Get the first pending request */ 414 req = list_first_entry(&sba->reqs_pending_list, 415 struct sba_request, node); 416 417 /* Try to make request active */ 418 if (!_sba_active_request(sba, req)) 419 break; 420 421 /* Send request to mailbox channel */ 422 ret = sba_send_mbox_request(sba, req); 423 if (ret < 0) { 424 _sba_pending_request(sba, req); 425 break; 426 } 427 428 count--; 429 } 430 } 431 432 static void sba_process_received_request(struct sba_device *sba, 433 struct sba_request *req) 434 { 435 unsigned long flags; 436 struct dma_async_tx_descriptor *tx; 437 struct sba_request *nreq, *first = req->first; 438 439 /* Process only after all chained requests are received */ 440 if (!atomic_dec_return(&first->next_pending_count)) { 441 tx = &first->tx; 442 443 WARN_ON(tx->cookie < 0); 444 if (tx->cookie > 0) { 445 dma_cookie_complete(tx); 446 dmaengine_desc_get_callback_invoke(tx, NULL); 447 dma_descriptor_unmap(tx); 448 tx->callback = NULL; 449 tx->callback_result = NULL; 450 } 451 452 dma_run_dependencies(tx); 453 454 spin_lock_irqsave(&sba->reqs_lock, flags); 455 456 /* Free all requests chained to first request */ 457 list_for_each_entry(nreq, &first->next, next) 458 _sba_free_request(sba, nreq); 459 INIT_LIST_HEAD(&first->next); 460 461 /* Free the first request */ 462 _sba_free_request(sba, first); 463 464 /* Process pending requests */ 465 _sba_process_pending_requests(sba); 466 467 spin_unlock_irqrestore(&sba->reqs_lock, flags); 468 } 469 } 470 471 static void sba_write_stats_in_seqfile(struct sba_device *sba, 472 struct seq_file *file) 473 { 474 unsigned long flags; 475 struct sba_request *req; 476 u32 free_count = 0, alloced_count = 0; 477 u32 pending_count = 0, active_count = 0, aborted_count = 0; 478 479 spin_lock_irqsave(&sba->reqs_lock, flags); 480 481 list_for_each_entry(req, &sba->reqs_free_list, node) 482 if (async_tx_test_ack(&req->tx)) 483 free_count++; 484 485 list_for_each_entry(req, &sba->reqs_alloc_list, node) 486 alloced_count++; 487 488 list_for_each_entry(req, &sba->reqs_pending_list, node) 489 pending_count++; 490 491 list_for_each_entry(req, &sba->reqs_active_list, node) 492 active_count++; 493 494 list_for_each_entry(req, &sba->reqs_aborted_list, node) 495 aborted_count++; 496 497 spin_unlock_irqrestore(&sba->reqs_lock, flags); 498 499 seq_printf(file, "maximum requests = %d\n", sba->max_req); 500 seq_printf(file, "free requests = %d\n", free_count); 501 seq_printf(file, "alloced requests = %d\n", alloced_count); 502 seq_printf(file, "pending requests = %d\n", pending_count); 503 seq_printf(file, "active requests = %d\n", active_count); 504 seq_printf(file, "aborted requests = %d\n", aborted_count); 505 } 506 507 /* ====== DMAENGINE callbacks ===== */ 508 509 static void sba_free_chan_resources(struct dma_chan *dchan) 510 { 511 /* 512 * Channel resources are pre-alloced so we just free-up 513 * whatever we can so that we can re-use pre-alloced 514 * channel resources next time. 515 */ 516 sba_cleanup_nonpending_requests(to_sba_device(dchan)); 517 } 518 519 static int sba_device_terminate_all(struct dma_chan *dchan) 520 { 521 /* Cleanup all pending requests */ 522 sba_cleanup_pending_requests(to_sba_device(dchan)); 523 524 return 0; 525 } 526 527 static void sba_issue_pending(struct dma_chan *dchan) 528 { 529 unsigned long flags; 530 struct sba_device *sba = to_sba_device(dchan); 531 532 /* Process pending requests */ 533 spin_lock_irqsave(&sba->reqs_lock, flags); 534 _sba_process_pending_requests(sba); 535 spin_unlock_irqrestore(&sba->reqs_lock, flags); 536 } 537 538 static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx) 539 { 540 unsigned long flags; 541 dma_cookie_t cookie; 542 struct sba_device *sba; 543 struct sba_request *req, *nreq; 544 545 if (unlikely(!tx)) 546 return -EINVAL; 547 548 sba = to_sba_device(tx->chan); 549 req = to_sba_request(tx); 550 551 /* Assign cookie and mark all chained requests pending */ 552 spin_lock_irqsave(&sba->reqs_lock, flags); 553 cookie = dma_cookie_assign(tx); 554 _sba_pending_request(sba, req); 555 list_for_each_entry(nreq, &req->next, next) 556 _sba_pending_request(sba, nreq); 557 spin_unlock_irqrestore(&sba->reqs_lock, flags); 558 559 return cookie; 560 } 561 562 static enum dma_status sba_tx_status(struct dma_chan *dchan, 563 dma_cookie_t cookie, 564 struct dma_tx_state *txstate) 565 { 566 enum dma_status ret; 567 struct sba_device *sba = to_sba_device(dchan); 568 569 ret = dma_cookie_status(dchan, cookie, txstate); 570 if (ret == DMA_COMPLETE) 571 return ret; 572 573 sba_peek_mchans(sba); 574 575 return dma_cookie_status(dchan, cookie, txstate); 576 } 577 578 static void sba_fillup_interrupt_msg(struct sba_request *req, 579 struct brcm_sba_command *cmds, 580 struct brcm_message *msg) 581 { 582 u64 cmd; 583 u32 c_mdata; 584 dma_addr_t resp_dma = req->tx.phys; 585 struct brcm_sba_command *cmdsp = cmds; 586 587 /* Type-B command to load dummy data into buf0 */ 588 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 589 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 590 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size, 591 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 592 c_mdata = sba_cmd_load_c_mdata(0); 593 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 594 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 595 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 596 SBA_CMD_SHIFT, SBA_CMD_MASK); 597 cmdsp->cmd = cmd; 598 *cmdsp->cmd_dma = cpu_to_le64(cmd); 599 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 600 cmdsp->data = resp_dma; 601 cmdsp->data_len = req->sba->hw_resp_size; 602 cmdsp++; 603 604 /* Type-A command to write buf0 to dummy location */ 605 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 606 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 607 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size, 608 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 609 cmd = sba_cmd_enc(cmd, 0x1, 610 SBA_RESP_SHIFT, SBA_RESP_MASK); 611 c_mdata = sba_cmd_write_c_mdata(0); 612 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 613 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 614 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 615 SBA_CMD_SHIFT, SBA_CMD_MASK); 616 cmdsp->cmd = cmd; 617 *cmdsp->cmd_dma = cpu_to_le64(cmd); 618 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 619 if (req->sba->hw_resp_size) { 620 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 621 cmdsp->resp = resp_dma; 622 cmdsp->resp_len = req->sba->hw_resp_size; 623 } 624 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 625 cmdsp->data = resp_dma; 626 cmdsp->data_len = req->sba->hw_resp_size; 627 cmdsp++; 628 629 /* Fillup brcm_message */ 630 msg->type = BRCM_MESSAGE_SBA; 631 msg->sba.cmds = cmds; 632 msg->sba.cmds_count = cmdsp - cmds; 633 msg->ctx = req; 634 msg->error = 0; 635 } 636 637 static struct dma_async_tx_descriptor * 638 sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags) 639 { 640 struct sba_request *req = NULL; 641 struct sba_device *sba = to_sba_device(dchan); 642 643 /* Alloc new request */ 644 req = sba_alloc_request(sba); 645 if (!req) 646 return NULL; 647 648 /* 649 * Force fence so that no requests are submitted 650 * until DMA callback for this request is invoked. 651 */ 652 req->flags |= SBA_REQUEST_FENCE; 653 654 /* Fillup request message */ 655 sba_fillup_interrupt_msg(req, req->cmds, &req->msg); 656 657 /* Init async_tx descriptor */ 658 req->tx.flags = flags; 659 req->tx.cookie = -EBUSY; 660 661 return &req->tx; 662 } 663 664 static void sba_fillup_memcpy_msg(struct sba_request *req, 665 struct brcm_sba_command *cmds, 666 struct brcm_message *msg, 667 dma_addr_t msg_offset, size_t msg_len, 668 dma_addr_t dst, dma_addr_t src) 669 { 670 u64 cmd; 671 u32 c_mdata; 672 dma_addr_t resp_dma = req->tx.phys; 673 struct brcm_sba_command *cmdsp = cmds; 674 675 /* Type-B command to load data into buf0 */ 676 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 677 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 678 cmd = sba_cmd_enc(cmd, msg_len, 679 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 680 c_mdata = sba_cmd_load_c_mdata(0); 681 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 682 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 683 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 684 SBA_CMD_SHIFT, SBA_CMD_MASK); 685 cmdsp->cmd = cmd; 686 *cmdsp->cmd_dma = cpu_to_le64(cmd); 687 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 688 cmdsp->data = src + msg_offset; 689 cmdsp->data_len = msg_len; 690 cmdsp++; 691 692 /* Type-A command to write buf0 */ 693 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 694 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 695 cmd = sba_cmd_enc(cmd, msg_len, 696 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 697 cmd = sba_cmd_enc(cmd, 0x1, 698 SBA_RESP_SHIFT, SBA_RESP_MASK); 699 c_mdata = sba_cmd_write_c_mdata(0); 700 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 701 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 702 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 703 SBA_CMD_SHIFT, SBA_CMD_MASK); 704 cmdsp->cmd = cmd; 705 *cmdsp->cmd_dma = cpu_to_le64(cmd); 706 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 707 if (req->sba->hw_resp_size) { 708 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 709 cmdsp->resp = resp_dma; 710 cmdsp->resp_len = req->sba->hw_resp_size; 711 } 712 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 713 cmdsp->data = dst + msg_offset; 714 cmdsp->data_len = msg_len; 715 cmdsp++; 716 717 /* Fillup brcm_message */ 718 msg->type = BRCM_MESSAGE_SBA; 719 msg->sba.cmds = cmds; 720 msg->sba.cmds_count = cmdsp - cmds; 721 msg->ctx = req; 722 msg->error = 0; 723 } 724 725 static struct sba_request * 726 sba_prep_dma_memcpy_req(struct sba_device *sba, 727 dma_addr_t off, dma_addr_t dst, dma_addr_t src, 728 size_t len, unsigned long flags) 729 { 730 struct sba_request *req = NULL; 731 732 /* Alloc new request */ 733 req = sba_alloc_request(sba); 734 if (!req) 735 return NULL; 736 if (flags & DMA_PREP_FENCE) 737 req->flags |= SBA_REQUEST_FENCE; 738 739 /* Fillup request message */ 740 sba_fillup_memcpy_msg(req, req->cmds, &req->msg, 741 off, len, dst, src); 742 743 /* Init async_tx descriptor */ 744 req->tx.flags = flags; 745 req->tx.cookie = -EBUSY; 746 747 return req; 748 } 749 750 static struct dma_async_tx_descriptor * 751 sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src, 752 size_t len, unsigned long flags) 753 { 754 size_t req_len; 755 dma_addr_t off = 0; 756 struct sba_device *sba = to_sba_device(dchan); 757 struct sba_request *first = NULL, *req; 758 759 /* Create chained requests where each request is upto hw_buf_size */ 760 while (len) { 761 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 762 763 req = sba_prep_dma_memcpy_req(sba, off, dst, src, 764 req_len, flags); 765 if (!req) { 766 if (first) 767 sba_free_chained_requests(first); 768 return NULL; 769 } 770 771 if (first) 772 sba_chain_request(first, req); 773 else 774 first = req; 775 776 off += req_len; 777 len -= req_len; 778 } 779 780 return (first) ? &first->tx : NULL; 781 } 782 783 static void sba_fillup_xor_msg(struct sba_request *req, 784 struct brcm_sba_command *cmds, 785 struct brcm_message *msg, 786 dma_addr_t msg_offset, size_t msg_len, 787 dma_addr_t dst, dma_addr_t *src, u32 src_cnt) 788 { 789 u64 cmd; 790 u32 c_mdata; 791 unsigned int i; 792 dma_addr_t resp_dma = req->tx.phys; 793 struct brcm_sba_command *cmdsp = cmds; 794 795 /* Type-B command to load data into buf0 */ 796 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 797 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 798 cmd = sba_cmd_enc(cmd, msg_len, 799 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 800 c_mdata = sba_cmd_load_c_mdata(0); 801 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 802 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 803 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 804 SBA_CMD_SHIFT, SBA_CMD_MASK); 805 cmdsp->cmd = cmd; 806 *cmdsp->cmd_dma = cpu_to_le64(cmd); 807 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 808 cmdsp->data = src[0] + msg_offset; 809 cmdsp->data_len = msg_len; 810 cmdsp++; 811 812 /* Type-B commands to xor data with buf0 and put it back in buf0 */ 813 for (i = 1; i < src_cnt; i++) { 814 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 815 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 816 cmd = sba_cmd_enc(cmd, msg_len, 817 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 818 c_mdata = sba_cmd_xor_c_mdata(0, 0); 819 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 820 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 821 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 822 SBA_CMD_SHIFT, SBA_CMD_MASK); 823 cmdsp->cmd = cmd; 824 *cmdsp->cmd_dma = cpu_to_le64(cmd); 825 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 826 cmdsp->data = src[i] + msg_offset; 827 cmdsp->data_len = msg_len; 828 cmdsp++; 829 } 830 831 /* Type-A command to write buf0 */ 832 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 833 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 834 cmd = sba_cmd_enc(cmd, msg_len, 835 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 836 cmd = sba_cmd_enc(cmd, 0x1, 837 SBA_RESP_SHIFT, SBA_RESP_MASK); 838 c_mdata = sba_cmd_write_c_mdata(0); 839 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 840 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 841 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 842 SBA_CMD_SHIFT, SBA_CMD_MASK); 843 cmdsp->cmd = cmd; 844 *cmdsp->cmd_dma = cpu_to_le64(cmd); 845 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 846 if (req->sba->hw_resp_size) { 847 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 848 cmdsp->resp = resp_dma; 849 cmdsp->resp_len = req->sba->hw_resp_size; 850 } 851 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 852 cmdsp->data = dst + msg_offset; 853 cmdsp->data_len = msg_len; 854 cmdsp++; 855 856 /* Fillup brcm_message */ 857 msg->type = BRCM_MESSAGE_SBA; 858 msg->sba.cmds = cmds; 859 msg->sba.cmds_count = cmdsp - cmds; 860 msg->ctx = req; 861 msg->error = 0; 862 } 863 864 static struct sba_request * 865 sba_prep_dma_xor_req(struct sba_device *sba, 866 dma_addr_t off, dma_addr_t dst, dma_addr_t *src, 867 u32 src_cnt, size_t len, unsigned long flags) 868 { 869 struct sba_request *req = NULL; 870 871 /* Alloc new request */ 872 req = sba_alloc_request(sba); 873 if (!req) 874 return NULL; 875 if (flags & DMA_PREP_FENCE) 876 req->flags |= SBA_REQUEST_FENCE; 877 878 /* Fillup request message */ 879 sba_fillup_xor_msg(req, req->cmds, &req->msg, 880 off, len, dst, src, src_cnt); 881 882 /* Init async_tx descriptor */ 883 req->tx.flags = flags; 884 req->tx.cookie = -EBUSY; 885 886 return req; 887 } 888 889 static struct dma_async_tx_descriptor * 890 sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src, 891 u32 src_cnt, size_t len, unsigned long flags) 892 { 893 size_t req_len; 894 dma_addr_t off = 0; 895 struct sba_device *sba = to_sba_device(dchan); 896 struct sba_request *first = NULL, *req; 897 898 /* Sanity checks */ 899 if (unlikely(src_cnt > sba->max_xor_srcs)) 900 return NULL; 901 902 /* Create chained requests where each request is upto hw_buf_size */ 903 while (len) { 904 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 905 906 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt, 907 req_len, flags); 908 if (!req) { 909 if (first) 910 sba_free_chained_requests(first); 911 return NULL; 912 } 913 914 if (first) 915 sba_chain_request(first, req); 916 else 917 first = req; 918 919 off += req_len; 920 len -= req_len; 921 } 922 923 return (first) ? &first->tx : NULL; 924 } 925 926 static void sba_fillup_pq_msg(struct sba_request *req, 927 bool pq_continue, 928 struct brcm_sba_command *cmds, 929 struct brcm_message *msg, 930 dma_addr_t msg_offset, size_t msg_len, 931 dma_addr_t *dst_p, dma_addr_t *dst_q, 932 const u8 *scf, dma_addr_t *src, u32 src_cnt) 933 { 934 u64 cmd; 935 u32 c_mdata; 936 unsigned int i; 937 dma_addr_t resp_dma = req->tx.phys; 938 struct brcm_sba_command *cmdsp = cmds; 939 940 if (pq_continue) { 941 /* Type-B command to load old P into buf0 */ 942 if (dst_p) { 943 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 944 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 945 cmd = sba_cmd_enc(cmd, msg_len, 946 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 947 c_mdata = sba_cmd_load_c_mdata(0); 948 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 949 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 950 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 951 SBA_CMD_SHIFT, SBA_CMD_MASK); 952 cmdsp->cmd = cmd; 953 *cmdsp->cmd_dma = cpu_to_le64(cmd); 954 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 955 cmdsp->data = *dst_p + msg_offset; 956 cmdsp->data_len = msg_len; 957 cmdsp++; 958 } 959 960 /* Type-B command to load old Q into buf1 */ 961 if (dst_q) { 962 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 963 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 964 cmd = sba_cmd_enc(cmd, msg_len, 965 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 966 c_mdata = sba_cmd_load_c_mdata(1); 967 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 968 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 969 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 970 SBA_CMD_SHIFT, SBA_CMD_MASK); 971 cmdsp->cmd = cmd; 972 *cmdsp->cmd_dma = cpu_to_le64(cmd); 973 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 974 cmdsp->data = *dst_q + msg_offset; 975 cmdsp->data_len = msg_len; 976 cmdsp++; 977 } 978 } else { 979 /* Type-A command to zero all buffers */ 980 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 981 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 982 cmd = sba_cmd_enc(cmd, msg_len, 983 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 984 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, 985 SBA_CMD_SHIFT, SBA_CMD_MASK); 986 cmdsp->cmd = cmd; 987 *cmdsp->cmd_dma = cpu_to_le64(cmd); 988 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 989 cmdsp++; 990 } 991 992 /* Type-B commands for generate P onto buf0 and Q onto buf1 */ 993 for (i = 0; i < src_cnt; i++) { 994 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 995 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 996 cmd = sba_cmd_enc(cmd, msg_len, 997 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 998 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0); 999 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1000 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1001 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1002 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1003 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR, 1004 SBA_CMD_SHIFT, SBA_CMD_MASK); 1005 cmdsp->cmd = cmd; 1006 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1007 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1008 cmdsp->data = src[i] + msg_offset; 1009 cmdsp->data_len = msg_len; 1010 cmdsp++; 1011 } 1012 1013 /* Type-A command to write buf0 */ 1014 if (dst_p) { 1015 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1016 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1017 cmd = sba_cmd_enc(cmd, msg_len, 1018 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1019 cmd = sba_cmd_enc(cmd, 0x1, 1020 SBA_RESP_SHIFT, SBA_RESP_MASK); 1021 c_mdata = sba_cmd_write_c_mdata(0); 1022 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1023 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1024 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1025 SBA_CMD_SHIFT, SBA_CMD_MASK); 1026 cmdsp->cmd = cmd; 1027 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1028 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1029 if (req->sba->hw_resp_size) { 1030 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1031 cmdsp->resp = resp_dma; 1032 cmdsp->resp_len = req->sba->hw_resp_size; 1033 } 1034 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1035 cmdsp->data = *dst_p + msg_offset; 1036 cmdsp->data_len = msg_len; 1037 cmdsp++; 1038 } 1039 1040 /* Type-A command to write buf1 */ 1041 if (dst_q) { 1042 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1043 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1044 cmd = sba_cmd_enc(cmd, msg_len, 1045 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1046 cmd = sba_cmd_enc(cmd, 0x1, 1047 SBA_RESP_SHIFT, SBA_RESP_MASK); 1048 c_mdata = sba_cmd_write_c_mdata(1); 1049 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1050 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1051 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1052 SBA_CMD_SHIFT, SBA_CMD_MASK); 1053 cmdsp->cmd = cmd; 1054 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1055 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1056 if (req->sba->hw_resp_size) { 1057 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1058 cmdsp->resp = resp_dma; 1059 cmdsp->resp_len = req->sba->hw_resp_size; 1060 } 1061 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1062 cmdsp->data = *dst_q + msg_offset; 1063 cmdsp->data_len = msg_len; 1064 cmdsp++; 1065 } 1066 1067 /* Fillup brcm_message */ 1068 msg->type = BRCM_MESSAGE_SBA; 1069 msg->sba.cmds = cmds; 1070 msg->sba.cmds_count = cmdsp - cmds; 1071 msg->ctx = req; 1072 msg->error = 0; 1073 } 1074 1075 static struct sba_request * 1076 sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off, 1077 dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src, 1078 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) 1079 { 1080 struct sba_request *req = NULL; 1081 1082 /* Alloc new request */ 1083 req = sba_alloc_request(sba); 1084 if (!req) 1085 return NULL; 1086 if (flags & DMA_PREP_FENCE) 1087 req->flags |= SBA_REQUEST_FENCE; 1088 1089 /* Fillup request messages */ 1090 sba_fillup_pq_msg(req, dmaf_continue(flags), 1091 req->cmds, &req->msg, 1092 off, len, dst_p, dst_q, scf, src, src_cnt); 1093 1094 /* Init async_tx descriptor */ 1095 req->tx.flags = flags; 1096 req->tx.cookie = -EBUSY; 1097 1098 return req; 1099 } 1100 1101 static void sba_fillup_pq_single_msg(struct sba_request *req, 1102 bool pq_continue, 1103 struct brcm_sba_command *cmds, 1104 struct brcm_message *msg, 1105 dma_addr_t msg_offset, size_t msg_len, 1106 dma_addr_t *dst_p, dma_addr_t *dst_q, 1107 dma_addr_t src, u8 scf) 1108 { 1109 u64 cmd; 1110 u32 c_mdata; 1111 u8 pos, dpos = raid6_gflog[scf]; 1112 dma_addr_t resp_dma = req->tx.phys; 1113 struct brcm_sba_command *cmdsp = cmds; 1114 1115 if (!dst_p) 1116 goto skip_p; 1117 1118 if (pq_continue) { 1119 /* Type-B command to load old P into buf0 */ 1120 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1121 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1122 cmd = sba_cmd_enc(cmd, msg_len, 1123 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1124 c_mdata = sba_cmd_load_c_mdata(0); 1125 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1126 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1127 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 1128 SBA_CMD_SHIFT, SBA_CMD_MASK); 1129 cmdsp->cmd = cmd; 1130 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1131 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1132 cmdsp->data = *dst_p + msg_offset; 1133 cmdsp->data_len = msg_len; 1134 cmdsp++; 1135 1136 /* 1137 * Type-B commands to xor data with buf0 and put it 1138 * back in buf0 1139 */ 1140 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1141 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1142 cmd = sba_cmd_enc(cmd, msg_len, 1143 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1144 c_mdata = sba_cmd_xor_c_mdata(0, 0); 1145 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1146 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1147 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 1148 SBA_CMD_SHIFT, SBA_CMD_MASK); 1149 cmdsp->cmd = cmd; 1150 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1151 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1152 cmdsp->data = src + msg_offset; 1153 cmdsp->data_len = msg_len; 1154 cmdsp++; 1155 } else { 1156 /* Type-B command to load old P into buf0 */ 1157 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1158 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1159 cmd = sba_cmd_enc(cmd, msg_len, 1160 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1161 c_mdata = sba_cmd_load_c_mdata(0); 1162 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1163 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1164 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 1165 SBA_CMD_SHIFT, SBA_CMD_MASK); 1166 cmdsp->cmd = cmd; 1167 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1168 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1169 cmdsp->data = src + msg_offset; 1170 cmdsp->data_len = msg_len; 1171 cmdsp++; 1172 } 1173 1174 /* Type-A command to write buf0 */ 1175 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1176 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1177 cmd = sba_cmd_enc(cmd, msg_len, 1178 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1179 cmd = sba_cmd_enc(cmd, 0x1, 1180 SBA_RESP_SHIFT, SBA_RESP_MASK); 1181 c_mdata = sba_cmd_write_c_mdata(0); 1182 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1183 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1184 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1185 SBA_CMD_SHIFT, SBA_CMD_MASK); 1186 cmdsp->cmd = cmd; 1187 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1188 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1189 if (req->sba->hw_resp_size) { 1190 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1191 cmdsp->resp = resp_dma; 1192 cmdsp->resp_len = req->sba->hw_resp_size; 1193 } 1194 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1195 cmdsp->data = *dst_p + msg_offset; 1196 cmdsp->data_len = msg_len; 1197 cmdsp++; 1198 1199 skip_p: 1200 if (!dst_q) 1201 goto skip_q; 1202 1203 /* Type-A command to zero all buffers */ 1204 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1205 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1206 cmd = sba_cmd_enc(cmd, msg_len, 1207 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1208 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, 1209 SBA_CMD_SHIFT, SBA_CMD_MASK); 1210 cmdsp->cmd = cmd; 1211 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1212 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1213 cmdsp++; 1214 1215 if (dpos == 255) 1216 goto skip_q_computation; 1217 pos = (dpos < req->sba->max_pq_coefs) ? 1218 dpos : (req->sba->max_pq_coefs - 1); 1219 1220 /* 1221 * Type-B command to generate initial Q from data 1222 * and store output into buf0 1223 */ 1224 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1225 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1226 cmd = sba_cmd_enc(cmd, msg_len, 1227 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1228 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0); 1229 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1230 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1231 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1232 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1233 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, 1234 SBA_CMD_SHIFT, SBA_CMD_MASK); 1235 cmdsp->cmd = cmd; 1236 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1237 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1238 cmdsp->data = src + msg_offset; 1239 cmdsp->data_len = msg_len; 1240 cmdsp++; 1241 1242 dpos -= pos; 1243 1244 /* Multiple Type-A command to generate final Q */ 1245 while (dpos) { 1246 pos = (dpos < req->sba->max_pq_coefs) ? 1247 dpos : (req->sba->max_pq_coefs - 1); 1248 1249 /* 1250 * Type-A command to generate Q with buf0 and 1251 * buf1 store result in buf0 1252 */ 1253 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1254 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1255 cmd = sba_cmd_enc(cmd, msg_len, 1256 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1257 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1); 1258 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1259 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1260 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1261 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1262 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, 1263 SBA_CMD_SHIFT, SBA_CMD_MASK); 1264 cmdsp->cmd = cmd; 1265 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1266 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1267 cmdsp++; 1268 1269 dpos -= pos; 1270 } 1271 1272 skip_q_computation: 1273 if (pq_continue) { 1274 /* 1275 * Type-B command to XOR previous output with 1276 * buf0 and write it into buf0 1277 */ 1278 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1279 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1280 cmd = sba_cmd_enc(cmd, msg_len, 1281 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1282 c_mdata = sba_cmd_xor_c_mdata(0, 0); 1283 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1284 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1285 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 1286 SBA_CMD_SHIFT, SBA_CMD_MASK); 1287 cmdsp->cmd = cmd; 1288 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1289 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1290 cmdsp->data = *dst_q + msg_offset; 1291 cmdsp->data_len = msg_len; 1292 cmdsp++; 1293 } 1294 1295 /* Type-A command to write buf0 */ 1296 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1297 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1298 cmd = sba_cmd_enc(cmd, msg_len, 1299 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1300 cmd = sba_cmd_enc(cmd, 0x1, 1301 SBA_RESP_SHIFT, SBA_RESP_MASK); 1302 c_mdata = sba_cmd_write_c_mdata(0); 1303 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1304 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1305 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1306 SBA_CMD_SHIFT, SBA_CMD_MASK); 1307 cmdsp->cmd = cmd; 1308 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1309 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1310 if (req->sba->hw_resp_size) { 1311 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1312 cmdsp->resp = resp_dma; 1313 cmdsp->resp_len = req->sba->hw_resp_size; 1314 } 1315 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1316 cmdsp->data = *dst_q + msg_offset; 1317 cmdsp->data_len = msg_len; 1318 cmdsp++; 1319 1320 skip_q: 1321 /* Fillup brcm_message */ 1322 msg->type = BRCM_MESSAGE_SBA; 1323 msg->sba.cmds = cmds; 1324 msg->sba.cmds_count = cmdsp - cmds; 1325 msg->ctx = req; 1326 msg->error = 0; 1327 } 1328 1329 static struct sba_request * 1330 sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off, 1331 dma_addr_t *dst_p, dma_addr_t *dst_q, 1332 dma_addr_t src, u8 scf, size_t len, 1333 unsigned long flags) 1334 { 1335 struct sba_request *req = NULL; 1336 1337 /* Alloc new request */ 1338 req = sba_alloc_request(sba); 1339 if (!req) 1340 return NULL; 1341 if (flags & DMA_PREP_FENCE) 1342 req->flags |= SBA_REQUEST_FENCE; 1343 1344 /* Fillup request messages */ 1345 sba_fillup_pq_single_msg(req, dmaf_continue(flags), 1346 req->cmds, &req->msg, off, len, 1347 dst_p, dst_q, src, scf); 1348 1349 /* Init async_tx descriptor */ 1350 req->tx.flags = flags; 1351 req->tx.cookie = -EBUSY; 1352 1353 return req; 1354 } 1355 1356 static struct dma_async_tx_descriptor * 1357 sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src, 1358 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) 1359 { 1360 u32 i, dst_q_index; 1361 size_t req_len; 1362 bool slow = false; 1363 dma_addr_t off = 0; 1364 dma_addr_t *dst_p = NULL, *dst_q = NULL; 1365 struct sba_device *sba = to_sba_device(dchan); 1366 struct sba_request *first = NULL, *req; 1367 1368 /* Sanity checks */ 1369 if (unlikely(src_cnt > sba->max_pq_srcs)) 1370 return NULL; 1371 for (i = 0; i < src_cnt; i++) 1372 if (sba->max_pq_coefs <= raid6_gflog[scf[i]]) 1373 slow = true; 1374 1375 /* Figure-out P and Q destination addresses */ 1376 if (!(flags & DMA_PREP_PQ_DISABLE_P)) 1377 dst_p = &dst[0]; 1378 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 1379 dst_q = &dst[1]; 1380 1381 /* Create chained requests where each request is upto hw_buf_size */ 1382 while (len) { 1383 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 1384 1385 if (slow) { 1386 dst_q_index = src_cnt; 1387 1388 if (dst_q) { 1389 for (i = 0; i < src_cnt; i++) { 1390 if (*dst_q == src[i]) { 1391 dst_q_index = i; 1392 break; 1393 } 1394 } 1395 } 1396 1397 if (dst_q_index < src_cnt) { 1398 i = dst_q_index; 1399 req = sba_prep_dma_pq_single_req(sba, 1400 off, dst_p, dst_q, src[i], scf[i], 1401 req_len, flags | DMA_PREP_FENCE); 1402 if (!req) 1403 goto fail; 1404 1405 if (first) 1406 sba_chain_request(first, req); 1407 else 1408 first = req; 1409 1410 flags |= DMA_PREP_CONTINUE; 1411 } 1412 1413 for (i = 0; i < src_cnt; i++) { 1414 if (dst_q_index == i) 1415 continue; 1416 1417 req = sba_prep_dma_pq_single_req(sba, 1418 off, dst_p, dst_q, src[i], scf[i], 1419 req_len, flags | DMA_PREP_FENCE); 1420 if (!req) 1421 goto fail; 1422 1423 if (first) 1424 sba_chain_request(first, req); 1425 else 1426 first = req; 1427 1428 flags |= DMA_PREP_CONTINUE; 1429 } 1430 } else { 1431 req = sba_prep_dma_pq_req(sba, off, 1432 dst_p, dst_q, src, src_cnt, 1433 scf, req_len, flags); 1434 if (!req) 1435 goto fail; 1436 1437 if (first) 1438 sba_chain_request(first, req); 1439 else 1440 first = req; 1441 } 1442 1443 off += req_len; 1444 len -= req_len; 1445 } 1446 1447 return (first) ? &first->tx : NULL; 1448 1449 fail: 1450 if (first) 1451 sba_free_chained_requests(first); 1452 return NULL; 1453 } 1454 1455 /* ====== Mailbox callbacks ===== */ 1456 1457 static void sba_receive_message(struct mbox_client *cl, void *msg) 1458 { 1459 struct brcm_message *m = msg; 1460 struct sba_request *req = m->ctx; 1461 struct sba_device *sba = req->sba; 1462 1463 /* Error count if message has error */ 1464 if (m->error < 0) 1465 dev_err(sba->dev, "%s got message with error %d", 1466 dma_chan_name(&sba->dma_chan), m->error); 1467 1468 /* Process received request */ 1469 sba_process_received_request(sba, req); 1470 } 1471 1472 /* ====== Debugfs callbacks ====== */ 1473 1474 static int sba_debugfs_stats_show(struct seq_file *file, void *offset) 1475 { 1476 struct platform_device *pdev = to_platform_device(file->private); 1477 struct sba_device *sba = platform_get_drvdata(pdev); 1478 1479 /* Write stats in file */ 1480 sba_write_stats_in_seqfile(sba, file); 1481 1482 return 0; 1483 } 1484 1485 /* ====== Platform driver routines ===== */ 1486 1487 static int sba_prealloc_channel_resources(struct sba_device *sba) 1488 { 1489 int i, j, ret = 0; 1490 struct sba_request *req = NULL; 1491 1492 sba->resp_base = dma_alloc_coherent(sba->mbox_dev, 1493 sba->max_resp_pool_size, 1494 &sba->resp_dma_base, GFP_KERNEL); 1495 if (!sba->resp_base) 1496 return -ENOMEM; 1497 1498 sba->cmds_base = dma_alloc_coherent(sba->mbox_dev, 1499 sba->max_cmds_pool_size, 1500 &sba->cmds_dma_base, GFP_KERNEL); 1501 if (!sba->cmds_base) { 1502 ret = -ENOMEM; 1503 goto fail_free_resp_pool; 1504 } 1505 1506 spin_lock_init(&sba->reqs_lock); 1507 sba->reqs_fence = false; 1508 INIT_LIST_HEAD(&sba->reqs_alloc_list); 1509 INIT_LIST_HEAD(&sba->reqs_pending_list); 1510 INIT_LIST_HEAD(&sba->reqs_active_list); 1511 INIT_LIST_HEAD(&sba->reqs_aborted_list); 1512 INIT_LIST_HEAD(&sba->reqs_free_list); 1513 1514 for (i = 0; i < sba->max_req; i++) { 1515 req = devm_kzalloc(sba->dev, 1516 sizeof(*req) + 1517 sba->max_cmd_per_req * sizeof(req->cmds[0]), 1518 GFP_KERNEL); 1519 if (!req) { 1520 ret = -ENOMEM; 1521 goto fail_free_cmds_pool; 1522 } 1523 INIT_LIST_HEAD(&req->node); 1524 req->sba = sba; 1525 req->flags = SBA_REQUEST_STATE_FREE; 1526 INIT_LIST_HEAD(&req->next); 1527 atomic_set(&req->next_pending_count, 0); 1528 for (j = 0; j < sba->max_cmd_per_req; j++) { 1529 req->cmds[j].cmd = 0; 1530 req->cmds[j].cmd_dma = sba->cmds_base + 1531 (i * sba->max_cmd_per_req + j) * sizeof(u64); 1532 req->cmds[j].cmd_dma_addr = sba->cmds_dma_base + 1533 (i * sba->max_cmd_per_req + j) * sizeof(u64); 1534 req->cmds[j].flags = 0; 1535 } 1536 memset(&req->msg, 0, sizeof(req->msg)); 1537 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); 1538 async_tx_ack(&req->tx); 1539 req->tx.tx_submit = sba_tx_submit; 1540 req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size; 1541 list_add_tail(&req->node, &sba->reqs_free_list); 1542 } 1543 1544 return 0; 1545 1546 fail_free_cmds_pool: 1547 dma_free_coherent(sba->mbox_dev, 1548 sba->max_cmds_pool_size, 1549 sba->cmds_base, sba->cmds_dma_base); 1550 fail_free_resp_pool: 1551 dma_free_coherent(sba->mbox_dev, 1552 sba->max_resp_pool_size, 1553 sba->resp_base, sba->resp_dma_base); 1554 return ret; 1555 } 1556 1557 static void sba_freeup_channel_resources(struct sba_device *sba) 1558 { 1559 dmaengine_terminate_all(&sba->dma_chan); 1560 dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size, 1561 sba->cmds_base, sba->cmds_dma_base); 1562 dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size, 1563 sba->resp_base, sba->resp_dma_base); 1564 sba->resp_base = NULL; 1565 sba->resp_dma_base = 0; 1566 } 1567 1568 static int sba_async_register(struct sba_device *sba) 1569 { 1570 int ret; 1571 struct dma_device *dma_dev = &sba->dma_dev; 1572 1573 /* Initialize DMA channel cookie */ 1574 sba->dma_chan.device = dma_dev; 1575 dma_cookie_init(&sba->dma_chan); 1576 1577 /* Initialize DMA device capability mask */ 1578 dma_cap_zero(dma_dev->cap_mask); 1579 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 1580 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 1581 dma_cap_set(DMA_XOR, dma_dev->cap_mask); 1582 dma_cap_set(DMA_PQ, dma_dev->cap_mask); 1583 1584 /* 1585 * Set mailbox channel device as the base device of 1586 * our dma_device because the actual memory accesses 1587 * will be done by mailbox controller 1588 */ 1589 dma_dev->dev = sba->mbox_dev; 1590 1591 /* Set base prep routines */ 1592 dma_dev->device_free_chan_resources = sba_free_chan_resources; 1593 dma_dev->device_terminate_all = sba_device_terminate_all; 1594 dma_dev->device_issue_pending = sba_issue_pending; 1595 dma_dev->device_tx_status = sba_tx_status; 1596 1597 /* Set interrupt routine */ 1598 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1599 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt; 1600 1601 /* Set memcpy routine */ 1602 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1603 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy; 1604 1605 /* Set xor routine and capability */ 1606 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1607 dma_dev->device_prep_dma_xor = sba_prep_dma_xor; 1608 dma_dev->max_xor = sba->max_xor_srcs; 1609 } 1610 1611 /* Set pq routine and capability */ 1612 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 1613 dma_dev->device_prep_dma_pq = sba_prep_dma_pq; 1614 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0); 1615 } 1616 1617 /* Initialize DMA device channel list */ 1618 INIT_LIST_HEAD(&dma_dev->channels); 1619 list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels); 1620 1621 /* Register with Linux async DMA framework*/ 1622 ret = dma_async_device_register(dma_dev); 1623 if (ret) { 1624 dev_err(sba->dev, "async device register error %d", ret); 1625 return ret; 1626 } 1627 1628 dev_info(sba->dev, "%s capabilities: %s%s%s%s\n", 1629 dma_chan_name(&sba->dma_chan), 1630 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "", 1631 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "", 1632 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1633 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : ""); 1634 1635 return 0; 1636 } 1637 1638 static int sba_probe(struct platform_device *pdev) 1639 { 1640 int i, ret = 0, mchans_count; 1641 struct sba_device *sba; 1642 struct platform_device *mbox_pdev; 1643 struct of_phandle_args args; 1644 1645 /* Allocate main SBA struct */ 1646 sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL); 1647 if (!sba) 1648 return -ENOMEM; 1649 1650 sba->dev = &pdev->dev; 1651 platform_set_drvdata(pdev, sba); 1652 1653 /* Number of channels equals number of mailbox channels */ 1654 ret = of_count_phandle_with_args(pdev->dev.of_node, 1655 "mboxes", "#mbox-cells"); 1656 if (ret <= 0) 1657 return -ENODEV; 1658 mchans_count = ret; 1659 1660 /* Determine SBA version from DT compatible string */ 1661 if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba")) 1662 sba->ver = SBA_VER_1; 1663 else if (of_device_is_compatible(sba->dev->of_node, 1664 "brcm,iproc-sba-v2")) 1665 sba->ver = SBA_VER_2; 1666 else 1667 return -ENODEV; 1668 1669 /* Derived Configuration parameters */ 1670 switch (sba->ver) { 1671 case SBA_VER_1: 1672 sba->hw_buf_size = 4096; 1673 sba->hw_resp_size = 8; 1674 sba->max_pq_coefs = 6; 1675 sba->max_pq_srcs = 6; 1676 break; 1677 case SBA_VER_2: 1678 sba->hw_buf_size = 4096; 1679 sba->hw_resp_size = 8; 1680 sba->max_pq_coefs = 30; 1681 /* 1682 * We can support max_pq_srcs == max_pq_coefs because 1683 * we are limited by number of SBA commands that we can 1684 * fit in one message for underlying ring manager HW. 1685 */ 1686 sba->max_pq_srcs = 12; 1687 break; 1688 default: 1689 return -EINVAL; 1690 } 1691 sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL * mchans_count; 1692 sba->max_cmd_per_req = sba->max_pq_srcs + 3; 1693 sba->max_xor_srcs = sba->max_cmd_per_req - 1; 1694 sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size; 1695 sba->max_cmds_pool_size = sba->max_req * 1696 sba->max_cmd_per_req * sizeof(u64); 1697 1698 /* Setup mailbox client */ 1699 sba->client.dev = &pdev->dev; 1700 sba->client.rx_callback = sba_receive_message; 1701 sba->client.tx_block = false; 1702 sba->client.knows_txdone = true; 1703 sba->client.tx_tout = 0; 1704 1705 /* Allocate mailbox channel array */ 1706 sba->mchans = devm_kcalloc(&pdev->dev, mchans_count, 1707 sizeof(*sba->mchans), GFP_KERNEL); 1708 if (!sba->mchans) 1709 return -ENOMEM; 1710 1711 /* Request mailbox channels */ 1712 sba->mchans_count = 0; 1713 for (i = 0; i < mchans_count; i++) { 1714 sba->mchans[i] = mbox_request_channel(&sba->client, i); 1715 if (IS_ERR(sba->mchans[i])) { 1716 ret = PTR_ERR(sba->mchans[i]); 1717 goto fail_free_mchans; 1718 } 1719 sba->mchans_count++; 1720 } 1721 atomic_set(&sba->mchans_current, 0); 1722 1723 /* Find-out underlying mailbox device */ 1724 ret = of_parse_phandle_with_args(pdev->dev.of_node, 1725 "mboxes", "#mbox-cells", 0, &args); 1726 if (ret) 1727 goto fail_free_mchans; 1728 mbox_pdev = of_find_device_by_node(args.np); 1729 of_node_put(args.np); 1730 if (!mbox_pdev) { 1731 ret = -ENODEV; 1732 goto fail_free_mchans; 1733 } 1734 sba->mbox_dev = &mbox_pdev->dev; 1735 1736 /* All mailbox channels should be of same ring manager device */ 1737 for (i = 1; i < mchans_count; i++) { 1738 ret = of_parse_phandle_with_args(pdev->dev.of_node, 1739 "mboxes", "#mbox-cells", i, &args); 1740 if (ret) 1741 goto fail_free_mchans; 1742 mbox_pdev = of_find_device_by_node(args.np); 1743 of_node_put(args.np); 1744 if (sba->mbox_dev != &mbox_pdev->dev) { 1745 ret = -EINVAL; 1746 goto fail_free_mchans; 1747 } 1748 } 1749 1750 /* Prealloc channel resource */ 1751 ret = sba_prealloc_channel_resources(sba); 1752 if (ret) 1753 goto fail_free_mchans; 1754 1755 /* Check availability of debugfs */ 1756 if (!debugfs_initialized()) 1757 goto skip_debugfs; 1758 1759 /* Create debugfs root entry */ 1760 sba->root = debugfs_create_dir(dev_name(sba->dev), NULL); 1761 if (IS_ERR_OR_NULL(sba->root)) { 1762 dev_err(sba->dev, "failed to create debugfs root entry\n"); 1763 sba->root = NULL; 1764 goto skip_debugfs; 1765 } 1766 1767 /* Create debugfs stats entry */ 1768 sba->stats = debugfs_create_devm_seqfile(sba->dev, "stats", sba->root, 1769 sba_debugfs_stats_show); 1770 if (IS_ERR_OR_NULL(sba->stats)) 1771 dev_err(sba->dev, "failed to create debugfs stats file\n"); 1772 skip_debugfs: 1773 1774 /* Register DMA device with Linux async framework */ 1775 ret = sba_async_register(sba); 1776 if (ret) 1777 goto fail_free_resources; 1778 1779 /* Print device info */ 1780 dev_info(sba->dev, "%s using SBAv%d and %d mailbox channels", 1781 dma_chan_name(&sba->dma_chan), sba->ver+1, 1782 sba->mchans_count); 1783 1784 return 0; 1785 1786 fail_free_resources: 1787 debugfs_remove_recursive(sba->root); 1788 sba_freeup_channel_resources(sba); 1789 fail_free_mchans: 1790 for (i = 0; i < sba->mchans_count; i++) 1791 mbox_free_channel(sba->mchans[i]); 1792 return ret; 1793 } 1794 1795 static int sba_remove(struct platform_device *pdev) 1796 { 1797 int i; 1798 struct sba_device *sba = platform_get_drvdata(pdev); 1799 1800 dma_async_device_unregister(&sba->dma_dev); 1801 1802 debugfs_remove_recursive(sba->root); 1803 1804 sba_freeup_channel_resources(sba); 1805 1806 for (i = 0; i < sba->mchans_count; i++) 1807 mbox_free_channel(sba->mchans[i]); 1808 1809 return 0; 1810 } 1811 1812 static const struct of_device_id sba_of_match[] = { 1813 { .compatible = "brcm,iproc-sba", }, 1814 { .compatible = "brcm,iproc-sba-v2", }, 1815 {}, 1816 }; 1817 MODULE_DEVICE_TABLE(of, sba_of_match); 1818 1819 static struct platform_driver sba_driver = { 1820 .probe = sba_probe, 1821 .remove = sba_remove, 1822 .driver = { 1823 .name = "bcm-sba-raid", 1824 .of_match_table = sba_of_match, 1825 }, 1826 }; 1827 module_platform_driver(sba_driver); 1828 1829 MODULE_DESCRIPTION("Broadcom SBA RAID driver"); 1830 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>"); 1831 MODULE_LICENSE("GPL v2"); 1832