1 /* 2 * Qualcomm Technologies HIDMA DMA engine low level code 3 * 4 * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 and 8 * only version 2 as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 */ 15 16 #include <linux/dmaengine.h> 17 #include <linux/slab.h> 18 #include <linux/interrupt.h> 19 #include <linux/mm.h> 20 #include <linux/highmem.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/delay.h> 23 #include <linux/atomic.h> 24 #include <linux/iopoll.h> 25 #include <linux/kfifo.h> 26 #include <linux/bitops.h> 27 28 #include "hidma.h" 29 30 #define HIDMA_EVRE_SIZE 16 /* each EVRE is 16 bytes */ 31 32 #define HIDMA_TRCA_CTRLSTS_REG 0x000 33 #define HIDMA_TRCA_RING_LOW_REG 0x008 34 #define HIDMA_TRCA_RING_HIGH_REG 0x00C 35 #define HIDMA_TRCA_RING_LEN_REG 0x010 36 #define HIDMA_TRCA_DOORBELL_REG 0x400 37 38 #define HIDMA_EVCA_CTRLSTS_REG 0x000 39 #define HIDMA_EVCA_INTCTRL_REG 0x004 40 #define HIDMA_EVCA_RING_LOW_REG 0x008 41 #define HIDMA_EVCA_RING_HIGH_REG 0x00C 42 #define HIDMA_EVCA_RING_LEN_REG 0x010 43 #define HIDMA_EVCA_WRITE_PTR_REG 0x020 44 #define HIDMA_EVCA_DOORBELL_REG 0x400 45 46 #define HIDMA_EVCA_IRQ_STAT_REG 0x100 47 #define HIDMA_EVCA_IRQ_CLR_REG 0x108 48 #define HIDMA_EVCA_IRQ_EN_REG 0x110 49 50 #define HIDMA_EVRE_CFG_IDX 0 51 52 #define HIDMA_EVRE_ERRINFO_BIT_POS 24 53 #define HIDMA_EVRE_CODE_BIT_POS 28 54 55 #define HIDMA_EVRE_ERRINFO_MASK GENMASK(3, 0) 56 #define HIDMA_EVRE_CODE_MASK GENMASK(3, 0) 57 58 #define HIDMA_CH_CONTROL_MASK GENMASK(7, 0) 59 #define HIDMA_CH_STATE_MASK GENMASK(7, 0) 60 #define HIDMA_CH_STATE_BIT_POS 0x8 61 62 #define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS 0 63 #define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS 1 64 #define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS 9 65 #define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS 10 66 #define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS 11 67 #define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS 14 68 69 #define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS) | \ 70 BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ 71 BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ 72 BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ 73 BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS) | \ 74 BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS)) 75 76 #define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size) \ 77 do { \ 78 iter += size; \ 79 if (iter >= ring_size) \ 80 iter -= ring_size; \ 81 } while (0) 82 83 #define HIDMA_CH_STATE(val) \ 84 ((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK) 85 86 #define HIDMA_ERR_INT_MASK \ 87 (BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS) | \ 88 BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ 89 BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ 90 BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ 91 BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS)) 92 93 enum ch_command { 94 HIDMA_CH_DISABLE = 0, 95 HIDMA_CH_ENABLE = 1, 96 HIDMA_CH_SUSPEND = 2, 97 HIDMA_CH_RESET = 9, 98 }; 99 100 enum ch_state { 101 HIDMA_CH_DISABLED = 0, 102 HIDMA_CH_ENABLED = 1, 103 HIDMA_CH_RUNNING = 2, 104 HIDMA_CH_SUSPENDED = 3, 105 HIDMA_CH_STOPPED = 4, 106 }; 107 108 enum tre_type { 109 HIDMA_TRE_MEMCPY = 3, 110 }; 111 112 enum err_code { 113 HIDMA_EVRE_STATUS_COMPLETE = 1, 114 HIDMA_EVRE_STATUS_ERROR = 4, 115 }; 116 117 static int hidma_is_chan_enabled(int state) 118 { 119 switch (state) { 120 case HIDMA_CH_ENABLED: 121 case HIDMA_CH_RUNNING: 122 return true; 123 default: 124 return false; 125 } 126 } 127 128 void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch) 129 { 130 struct hidma_tre *tre; 131 132 if (tre_ch >= lldev->nr_tres) { 133 dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch); 134 return; 135 } 136 137 tre = &lldev->trepool[tre_ch]; 138 if (atomic_read(&tre->allocated) != true) { 139 dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch); 140 return; 141 } 142 143 atomic_set(&tre->allocated, 0); 144 } 145 146 int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name, 147 void (*callback)(void *data), void *data, u32 *tre_ch) 148 { 149 unsigned int i; 150 struct hidma_tre *tre; 151 u32 *tre_local; 152 153 if (!tre_ch || !lldev) 154 return -EINVAL; 155 156 /* need to have at least one empty spot in the queue */ 157 for (i = 0; i < lldev->nr_tres - 1; i++) { 158 if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1)) 159 break; 160 } 161 162 if (i == (lldev->nr_tres - 1)) 163 return -ENOMEM; 164 165 tre = &lldev->trepool[i]; 166 tre->dma_sig = sig; 167 tre->dev_name = dev_name; 168 tre->callback = callback; 169 tre->data = data; 170 tre->idx = i; 171 tre->status = 0; 172 tre->queued = 0; 173 tre->err_code = 0; 174 tre->err_info = 0; 175 tre->lldev = lldev; 176 tre_local = &tre->tre_local[0]; 177 tre_local[HIDMA_TRE_CFG_IDX] = HIDMA_TRE_MEMCPY; 178 tre_local[HIDMA_TRE_CFG_IDX] |= (lldev->chidx & 0xFF) << 8; 179 tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16); /* set IEOB */ 180 *tre_ch = i; 181 if (callback) 182 callback(data); 183 return 0; 184 } 185 186 /* 187 * Multiple TREs may be queued and waiting in the pending queue. 188 */ 189 static void hidma_ll_tre_complete(unsigned long arg) 190 { 191 struct hidma_lldev *lldev = (struct hidma_lldev *)arg; 192 struct hidma_tre *tre; 193 194 while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) { 195 /* call the user if it has been read by the hardware */ 196 if (tre->callback) 197 tre->callback(tre->data); 198 } 199 } 200 201 static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info, 202 u8 err_code) 203 { 204 struct hidma_tre *tre; 205 unsigned long flags; 206 u32 tre_iterator; 207 208 spin_lock_irqsave(&lldev->lock, flags); 209 210 tre_iterator = lldev->tre_processed_off; 211 tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE]; 212 if (!tre) { 213 spin_unlock_irqrestore(&lldev->lock, flags); 214 dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n", 215 tre_iterator / HIDMA_TRE_SIZE); 216 return -EINVAL; 217 } 218 lldev->pending_tre_list[tre->tre_index] = NULL; 219 220 /* 221 * Keep track of pending TREs that SW is expecting to receive 222 * from HW. We got one now. Decrement our counter. 223 */ 224 if (atomic_dec_return(&lldev->pending_tre_count) < 0) { 225 dev_warn(lldev->dev, "tre count mismatch on completion"); 226 atomic_set(&lldev->pending_tre_count, 0); 227 } 228 229 HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, 230 lldev->tre_ring_size); 231 lldev->tre_processed_off = tre_iterator; 232 spin_unlock_irqrestore(&lldev->lock, flags); 233 234 tre->err_info = err_info; 235 tre->err_code = err_code; 236 tre->queued = 0; 237 238 kfifo_put(&lldev->handoff_fifo, tre); 239 tasklet_schedule(&lldev->task); 240 241 return 0; 242 } 243 244 /* 245 * Called to handle the interrupt for the channel. 246 * Return a positive number if TRE or EVRE were consumed on this run. 247 * Return a positive number if there are pending TREs or EVREs. 248 * Return 0 if there is nothing to consume or no pending TREs/EVREs found. 249 */ 250 static int hidma_handle_tre_completion(struct hidma_lldev *lldev) 251 { 252 u32 evre_ring_size = lldev->evre_ring_size; 253 u32 err_info, err_code, evre_write_off; 254 u32 evre_iterator; 255 u32 num_completed = 0; 256 257 evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); 258 evre_iterator = lldev->evre_processed_off; 259 260 if ((evre_write_off > evre_ring_size) || 261 (evre_write_off % HIDMA_EVRE_SIZE)) { 262 dev_err(lldev->dev, "HW reports invalid EVRE write offset\n"); 263 return 0; 264 } 265 266 /* 267 * By the time control reaches here the number of EVREs and TREs 268 * may not match. Only consume the ones that hardware told us. 269 */ 270 while ((evre_iterator != evre_write_off)) { 271 u32 *current_evre = lldev->evre_ring + evre_iterator; 272 u32 cfg; 273 274 cfg = current_evre[HIDMA_EVRE_CFG_IDX]; 275 err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS; 276 err_info &= HIDMA_EVRE_ERRINFO_MASK; 277 err_code = 278 (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK; 279 280 if (hidma_post_completed(lldev, err_info, err_code)) 281 break; 282 283 HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE, 284 evre_ring_size); 285 286 /* 287 * Read the new event descriptor written by the HW. 288 * As we are processing the delivered events, other events 289 * get queued to the SW for processing. 290 */ 291 evre_write_off = 292 readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); 293 num_completed++; 294 295 /* 296 * An error interrupt might have arrived while we are processing 297 * the completed interrupt. 298 */ 299 if (!hidma_ll_isenabled(lldev)) 300 break; 301 } 302 303 if (num_completed) { 304 u32 evre_read_off = (lldev->evre_processed_off + 305 HIDMA_EVRE_SIZE * num_completed); 306 evre_read_off = evre_read_off % evre_ring_size; 307 writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG); 308 309 /* record the last processed tre offset */ 310 lldev->evre_processed_off = evre_read_off; 311 } 312 313 return num_completed; 314 } 315 316 void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info, 317 u8 err_code) 318 { 319 while (atomic_read(&lldev->pending_tre_count)) { 320 if (hidma_post_completed(lldev, err_info, err_code)) 321 break; 322 } 323 } 324 325 static int hidma_ll_reset(struct hidma_lldev *lldev) 326 { 327 u32 val; 328 int ret; 329 330 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 331 val &= ~(HIDMA_CH_CONTROL_MASK << 16); 332 val |= HIDMA_CH_RESET << 16; 333 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 334 335 /* 336 * Delay 10ms after reset to allow DMA logic to quiesce. 337 * Do a polled read up to 1ms and 10ms maximum. 338 */ 339 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, 340 HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, 341 1000, 10000); 342 if (ret) { 343 dev_err(lldev->dev, "transfer channel did not reset\n"); 344 return ret; 345 } 346 347 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 348 val &= ~(HIDMA_CH_CONTROL_MASK << 16); 349 val |= HIDMA_CH_RESET << 16; 350 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 351 352 /* 353 * Delay 10ms after reset to allow DMA logic to quiesce. 354 * Do a polled read up to 1ms and 10ms maximum. 355 */ 356 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, 357 HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, 358 1000, 10000); 359 if (ret) 360 return ret; 361 362 lldev->trch_state = HIDMA_CH_DISABLED; 363 lldev->evch_state = HIDMA_CH_DISABLED; 364 return 0; 365 } 366 367 /* 368 * The interrupt handler for HIDMA will try to consume as many pending 369 * EVRE from the event queue as possible. Each EVRE has an associated 370 * TRE that holds the user interface parameters. EVRE reports the 371 * result of the transaction. Hardware guarantees ordering between EVREs 372 * and TREs. We use last processed offset to figure out which TRE is 373 * associated with which EVRE. If two TREs are consumed by HW, the EVREs 374 * are in order in the event ring. 375 * 376 * This handler will do a one pass for consuming EVREs. Other EVREs may 377 * be delivered while we are working. It will try to consume incoming 378 * EVREs one more time and return. 379 * 380 * For unprocessed EVREs, hardware will trigger another interrupt until 381 * all the interrupt bits are cleared. 382 * 383 * Hardware guarantees that by the time interrupt is observed, all data 384 * transactions in flight are delivered to their respective places and 385 * are visible to the CPU. 386 * 387 * On demand paging for IOMMU is only supported for PCIe via PRI 388 * (Page Request Interface) not for HIDMA. All other hardware instances 389 * including HIDMA work on pinned DMA addresses. 390 * 391 * HIDMA is not aware of IOMMU presence since it follows the DMA API. All 392 * IOMMU latency will be built into the data movement time. By the time 393 * interrupt happens, IOMMU lookups + data movement has already taken place. 394 * 395 * While the first read in a typical PCI endpoint ISR flushes all outstanding 396 * requests traditionally to the destination, this concept does not apply 397 * here for this HW. 398 */ 399 static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause) 400 { 401 if (cause & HIDMA_ERR_INT_MASK) { 402 dev_err(lldev->dev, "error 0x%x, disabling...\n", 403 cause); 404 405 /* Clear out pending interrupts */ 406 writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 407 408 /* No further submissions. */ 409 hidma_ll_disable(lldev); 410 411 /* Driver completes the txn and intimates the client.*/ 412 hidma_cleanup_pending_tre(lldev, 0xFF, 413 HIDMA_EVRE_STATUS_ERROR); 414 415 return; 416 } 417 418 /* 419 * Fine tuned for this HW... 420 * 421 * This ISR has been designed for this particular hardware. Relaxed 422 * read and write accessors are used for performance reasons due to 423 * interrupt delivery guarantees. Do not copy this code blindly and 424 * expect that to work. 425 * 426 * Try to consume as many EVREs as possible. 427 */ 428 hidma_handle_tre_completion(lldev); 429 430 /* We consumed TREs or there are pending TREs or EVREs. */ 431 writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 432 } 433 434 irqreturn_t hidma_ll_inthandler(int chirq, void *arg) 435 { 436 struct hidma_lldev *lldev = arg; 437 u32 status; 438 u32 enable; 439 u32 cause; 440 441 status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 442 enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 443 cause = status & enable; 444 445 while (cause) { 446 hidma_ll_int_handler_internal(lldev, cause); 447 448 /* 449 * Another interrupt might have arrived while we are 450 * processing this one. Read the new cause. 451 */ 452 status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 453 enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 454 cause = status & enable; 455 } 456 457 return IRQ_HANDLED; 458 } 459 460 irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause) 461 { 462 struct hidma_lldev *lldev = arg; 463 464 hidma_ll_int_handler_internal(lldev, cause); 465 return IRQ_HANDLED; 466 } 467 468 int hidma_ll_enable(struct hidma_lldev *lldev) 469 { 470 u32 val; 471 int ret; 472 473 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 474 val &= ~(HIDMA_CH_CONTROL_MASK << 16); 475 val |= HIDMA_CH_ENABLE << 16; 476 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 477 478 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, 479 hidma_is_chan_enabled(HIDMA_CH_STATE(val)), 480 1000, 10000); 481 if (ret) { 482 dev_err(lldev->dev, "event channel did not get enabled\n"); 483 return ret; 484 } 485 486 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 487 val &= ~(HIDMA_CH_CONTROL_MASK << 16); 488 val |= HIDMA_CH_ENABLE << 16; 489 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 490 491 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, 492 hidma_is_chan_enabled(HIDMA_CH_STATE(val)), 493 1000, 10000); 494 if (ret) { 495 dev_err(lldev->dev, "transfer channel did not get enabled\n"); 496 return ret; 497 } 498 499 lldev->trch_state = HIDMA_CH_ENABLED; 500 lldev->evch_state = HIDMA_CH_ENABLED; 501 502 return 0; 503 } 504 505 void hidma_ll_start(struct hidma_lldev *lldev) 506 { 507 unsigned long irqflags; 508 509 spin_lock_irqsave(&lldev->lock, irqflags); 510 writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG); 511 spin_unlock_irqrestore(&lldev->lock, irqflags); 512 } 513 514 bool hidma_ll_isenabled(struct hidma_lldev *lldev) 515 { 516 u32 val; 517 518 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 519 lldev->trch_state = HIDMA_CH_STATE(val); 520 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 521 lldev->evch_state = HIDMA_CH_STATE(val); 522 523 /* both channels have to be enabled before calling this function */ 524 if (hidma_is_chan_enabled(lldev->trch_state) && 525 hidma_is_chan_enabled(lldev->evch_state)) 526 return true; 527 528 return false; 529 } 530 531 void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch) 532 { 533 struct hidma_tre *tre; 534 unsigned long flags; 535 536 tre = &lldev->trepool[tre_ch]; 537 538 /* copy the TRE into its location in the TRE ring */ 539 spin_lock_irqsave(&lldev->lock, flags); 540 tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE; 541 lldev->pending_tre_list[tre->tre_index] = tre; 542 memcpy(lldev->tre_ring + lldev->tre_write_offset, 543 &tre->tre_local[0], HIDMA_TRE_SIZE); 544 tre->err_code = 0; 545 tre->err_info = 0; 546 tre->queued = 1; 547 atomic_inc(&lldev->pending_tre_count); 548 lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE) 549 % lldev->tre_ring_size; 550 spin_unlock_irqrestore(&lldev->lock, flags); 551 } 552 553 /* 554 * Note that even though we stop this channel if there is a pending transaction 555 * in flight it will complete and follow the callback. This request will 556 * prevent further requests to be made. 557 */ 558 int hidma_ll_disable(struct hidma_lldev *lldev) 559 { 560 u32 val; 561 int ret; 562 563 /* The channel needs to be in working state */ 564 if (!hidma_ll_isenabled(lldev)) 565 return 0; 566 567 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 568 val &= ~(HIDMA_CH_CONTROL_MASK << 16); 569 val |= HIDMA_CH_SUSPEND << 16; 570 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 571 572 /* 573 * Start the wait right after the suspend is confirmed. 574 * Do a polled read up to 1ms and 10ms maximum. 575 */ 576 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, 577 HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, 578 1000, 10000); 579 if (ret) 580 return ret; 581 582 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 583 val &= ~(HIDMA_CH_CONTROL_MASK << 16); 584 val |= HIDMA_CH_SUSPEND << 16; 585 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 586 587 /* 588 * Start the wait right after the suspend is confirmed 589 * Delay up to 10ms after reset to allow DMA logic to quiesce. 590 */ 591 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, 592 HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, 593 1000, 10000); 594 if (ret) 595 return ret; 596 597 lldev->trch_state = HIDMA_CH_SUSPENDED; 598 lldev->evch_state = HIDMA_CH_SUSPENDED; 599 return 0; 600 } 601 602 void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch, 603 dma_addr_t src, dma_addr_t dest, u32 len, 604 u32 flags) 605 { 606 struct hidma_tre *tre; 607 u32 *tre_local; 608 609 if (tre_ch >= lldev->nr_tres) { 610 dev_err(lldev->dev, "invalid TRE number in transfer params:%d", 611 tre_ch); 612 return; 613 } 614 615 tre = &lldev->trepool[tre_ch]; 616 if (atomic_read(&tre->allocated) != true) { 617 dev_err(lldev->dev, "trying to set params on an unused TRE:%d", 618 tre_ch); 619 return; 620 } 621 622 tre_local = &tre->tre_local[0]; 623 tre_local[HIDMA_TRE_LEN_IDX] = len; 624 tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src); 625 tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src); 626 tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest); 627 tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest); 628 tre->int_flags = flags; 629 } 630 631 /* 632 * Called during initialization and after an error condition 633 * to restore hardware state. 634 */ 635 int hidma_ll_setup(struct hidma_lldev *lldev) 636 { 637 int rc; 638 u64 addr; 639 u32 val; 640 u32 nr_tres = lldev->nr_tres; 641 642 atomic_set(&lldev->pending_tre_count, 0); 643 lldev->tre_processed_off = 0; 644 lldev->evre_processed_off = 0; 645 lldev->tre_write_offset = 0; 646 647 /* disable interrupts */ 648 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 649 650 /* clear all pending interrupts */ 651 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 652 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 653 654 rc = hidma_ll_reset(lldev); 655 if (rc) 656 return rc; 657 658 /* 659 * Clear all pending interrupts again. 660 * Otherwise, we observe reset complete interrupts. 661 */ 662 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 663 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 664 665 /* disable interrupts again after reset */ 666 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 667 668 addr = lldev->tre_dma; 669 writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG); 670 writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG); 671 writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG); 672 673 addr = lldev->evre_dma; 674 writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG); 675 writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG); 676 writel(HIDMA_EVRE_SIZE * nr_tres, 677 lldev->evca + HIDMA_EVCA_RING_LEN_REG); 678 679 /* configure interrupts */ 680 hidma_ll_setup_irq(lldev, lldev->msi_support); 681 682 rc = hidma_ll_enable(lldev); 683 if (rc) 684 return rc; 685 686 return rc; 687 } 688 689 void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi) 690 { 691 u32 val; 692 693 lldev->msi_support = msi; 694 695 /* disable interrupts again after reset */ 696 writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 697 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 698 699 /* support IRQ by default */ 700 val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG); 701 val &= ~0xF; 702 if (!lldev->msi_support) 703 val = val | 0x1; 704 writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG); 705 706 /* clear all pending interrupts and enable them */ 707 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 708 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 709 } 710 711 struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres, 712 void __iomem *trca, void __iomem *evca, 713 u8 chidx) 714 { 715 u32 required_bytes; 716 struct hidma_lldev *lldev; 717 int rc; 718 size_t sz; 719 720 if (!trca || !evca || !dev || !nr_tres) 721 return NULL; 722 723 /* need at least four TREs */ 724 if (nr_tres < 4) 725 return NULL; 726 727 /* need an extra space */ 728 nr_tres += 1; 729 730 lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL); 731 if (!lldev) 732 return NULL; 733 734 lldev->evca = evca; 735 lldev->trca = trca; 736 lldev->dev = dev; 737 sz = sizeof(struct hidma_tre); 738 lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL); 739 if (!lldev->trepool) 740 return NULL; 741 742 required_bytes = sizeof(lldev->pending_tre_list[0]); 743 lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes, 744 GFP_KERNEL); 745 if (!lldev->pending_tre_list) 746 return NULL; 747 748 sz = (HIDMA_TRE_SIZE + 1) * nr_tres; 749 lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma, 750 GFP_KERNEL); 751 if (!lldev->tre_ring) 752 return NULL; 753 754 memset(lldev->tre_ring, 0, (HIDMA_TRE_SIZE + 1) * nr_tres); 755 lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres; 756 lldev->nr_tres = nr_tres; 757 758 /* the TRE ring has to be TRE_SIZE aligned */ 759 if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) { 760 u8 tre_ring_shift; 761 762 tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE; 763 tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift; 764 lldev->tre_dma += tre_ring_shift; 765 lldev->tre_ring += tre_ring_shift; 766 } 767 768 sz = (HIDMA_EVRE_SIZE + 1) * nr_tres; 769 lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma, 770 GFP_KERNEL); 771 if (!lldev->evre_ring) 772 return NULL; 773 774 memset(lldev->evre_ring, 0, (HIDMA_EVRE_SIZE + 1) * nr_tres); 775 lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres; 776 777 /* the EVRE ring has to be EVRE_SIZE aligned */ 778 if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) { 779 u8 evre_ring_shift; 780 781 evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE; 782 evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift; 783 lldev->evre_dma += evre_ring_shift; 784 lldev->evre_ring += evre_ring_shift; 785 } 786 lldev->nr_tres = nr_tres; 787 lldev->chidx = chidx; 788 789 sz = nr_tres * sizeof(struct hidma_tre *); 790 rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL); 791 if (rc) 792 return NULL; 793 794 rc = hidma_ll_setup(lldev); 795 if (rc) 796 return NULL; 797 798 spin_lock_init(&lldev->lock); 799 tasklet_init(&lldev->task, hidma_ll_tre_complete, (unsigned long)lldev); 800 lldev->initialized = 1; 801 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 802 return lldev; 803 } 804 805 int hidma_ll_uninit(struct hidma_lldev *lldev) 806 { 807 u32 required_bytes; 808 int rc = 0; 809 u32 val; 810 811 if (!lldev) 812 return -ENODEV; 813 814 if (!lldev->initialized) 815 return 0; 816 817 lldev->initialized = 0; 818 819 required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres; 820 tasklet_kill(&lldev->task); 821 memset(lldev->trepool, 0, required_bytes); 822 lldev->trepool = NULL; 823 atomic_set(&lldev->pending_tre_count, 0); 824 lldev->tre_write_offset = 0; 825 826 rc = hidma_ll_reset(lldev); 827 828 /* 829 * Clear all pending interrupts again. 830 * Otherwise, we observe reset complete interrupts. 831 */ 832 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 833 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 834 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 835 return rc; 836 } 837 838 enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch) 839 { 840 enum dma_status ret = DMA_ERROR; 841 struct hidma_tre *tre; 842 unsigned long flags; 843 u8 err_code; 844 845 spin_lock_irqsave(&lldev->lock, flags); 846 847 tre = &lldev->trepool[tre_ch]; 848 err_code = tre->err_code; 849 850 if (err_code & HIDMA_EVRE_STATUS_COMPLETE) 851 ret = DMA_COMPLETE; 852 else if (err_code & HIDMA_EVRE_STATUS_ERROR) 853 ret = DMA_ERROR; 854 else 855 ret = DMA_IN_PROGRESS; 856 spin_unlock_irqrestore(&lldev->lock, flags); 857 858 return ret; 859 } 860