1d1615ca2SSinan Kaya /* 2d1615ca2SSinan Kaya * Qualcomm Technologies HIDMA DMA engine low level code 3d1615ca2SSinan Kaya * 4d1615ca2SSinan Kaya * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved. 5d1615ca2SSinan Kaya * 6d1615ca2SSinan Kaya * This program is free software; you can redistribute it and/or modify 7d1615ca2SSinan Kaya * it under the terms of the GNU General Public License version 2 and 8d1615ca2SSinan Kaya * only version 2 as published by the Free Software Foundation. 9d1615ca2SSinan Kaya * 10d1615ca2SSinan Kaya * This program is distributed in the hope that it will be useful, 11d1615ca2SSinan Kaya * but WITHOUT ANY WARRANTY; without even the implied warranty of 12d1615ca2SSinan Kaya * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13d1615ca2SSinan Kaya * GNU General Public License for more details. 14d1615ca2SSinan Kaya */ 15d1615ca2SSinan Kaya 16d1615ca2SSinan Kaya #include <linux/dmaengine.h> 17d1615ca2SSinan Kaya #include <linux/slab.h> 18d1615ca2SSinan Kaya #include <linux/interrupt.h> 19d1615ca2SSinan Kaya #include <linux/mm.h> 20d1615ca2SSinan Kaya #include <linux/highmem.h> 21d1615ca2SSinan Kaya #include <linux/dma-mapping.h> 22d1615ca2SSinan Kaya #include <linux/delay.h> 23d1615ca2SSinan Kaya #include <linux/atomic.h> 24d1615ca2SSinan Kaya #include <linux/iopoll.h> 25d1615ca2SSinan Kaya #include <linux/kfifo.h> 26d1615ca2SSinan Kaya #include <linux/bitops.h> 27d1615ca2SSinan Kaya 28d1615ca2SSinan Kaya #include "hidma.h" 29d1615ca2SSinan Kaya 30d1615ca2SSinan Kaya #define HIDMA_EVRE_SIZE 16 /* each EVRE is 16 bytes */ 31d1615ca2SSinan Kaya 32d1615ca2SSinan Kaya #define HIDMA_TRCA_CTRLSTS_REG 0x000 33d1615ca2SSinan Kaya #define HIDMA_TRCA_RING_LOW_REG 0x008 34d1615ca2SSinan Kaya #define HIDMA_TRCA_RING_HIGH_REG 0x00C 35d1615ca2SSinan Kaya #define HIDMA_TRCA_RING_LEN_REG 0x010 36d1615ca2SSinan Kaya #define HIDMA_TRCA_DOORBELL_REG 0x400 37d1615ca2SSinan Kaya 38d1615ca2SSinan Kaya #define HIDMA_EVCA_CTRLSTS_REG 0x000 39d1615ca2SSinan Kaya #define HIDMA_EVCA_INTCTRL_REG 0x004 40d1615ca2SSinan Kaya #define HIDMA_EVCA_RING_LOW_REG 0x008 41d1615ca2SSinan Kaya #define HIDMA_EVCA_RING_HIGH_REG 0x00C 42d1615ca2SSinan Kaya #define HIDMA_EVCA_RING_LEN_REG 0x010 43d1615ca2SSinan Kaya #define HIDMA_EVCA_WRITE_PTR_REG 0x020 44d1615ca2SSinan Kaya #define HIDMA_EVCA_DOORBELL_REG 0x400 45d1615ca2SSinan Kaya 46d1615ca2SSinan Kaya #define HIDMA_EVCA_IRQ_STAT_REG 0x100 47d1615ca2SSinan Kaya #define HIDMA_EVCA_IRQ_CLR_REG 0x108 48d1615ca2SSinan Kaya #define HIDMA_EVCA_IRQ_EN_REG 0x110 49d1615ca2SSinan Kaya 50d1615ca2SSinan Kaya #define HIDMA_EVRE_CFG_IDX 0 51d1615ca2SSinan Kaya 52d1615ca2SSinan Kaya #define HIDMA_EVRE_ERRINFO_BIT_POS 24 53d1615ca2SSinan Kaya #define HIDMA_EVRE_CODE_BIT_POS 28 54d1615ca2SSinan Kaya 55d1615ca2SSinan Kaya #define HIDMA_EVRE_ERRINFO_MASK GENMASK(3, 0) 56d1615ca2SSinan Kaya #define HIDMA_EVRE_CODE_MASK GENMASK(3, 0) 57d1615ca2SSinan Kaya 58d1615ca2SSinan Kaya #define HIDMA_CH_CONTROL_MASK GENMASK(7, 0) 59d1615ca2SSinan Kaya #define HIDMA_CH_STATE_MASK GENMASK(7, 0) 60d1615ca2SSinan Kaya #define HIDMA_CH_STATE_BIT_POS 0x8 61d1615ca2SSinan Kaya 62d1615ca2SSinan Kaya #define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS 0 63d1615ca2SSinan Kaya #define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS 1 64d1615ca2SSinan Kaya #define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS 9 65d1615ca2SSinan Kaya #define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS 10 66d1615ca2SSinan Kaya #define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS 11 67d1615ca2SSinan Kaya #define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS 14 68d1615ca2SSinan Kaya 69d1615ca2SSinan Kaya #define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS) | \ 70d1615ca2SSinan Kaya BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ 71d1615ca2SSinan Kaya BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ 72d1615ca2SSinan Kaya BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ 73d1615ca2SSinan Kaya BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS) | \ 74d1615ca2SSinan Kaya BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS)) 75d1615ca2SSinan Kaya 76d1615ca2SSinan Kaya #define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size) \ 77d1615ca2SSinan Kaya do { \ 78d1615ca2SSinan Kaya iter += size; \ 79d1615ca2SSinan Kaya if (iter >= ring_size) \ 80d1615ca2SSinan Kaya iter -= ring_size; \ 81d1615ca2SSinan Kaya } while (0) 82d1615ca2SSinan Kaya 83d1615ca2SSinan Kaya #define HIDMA_CH_STATE(val) \ 84d1615ca2SSinan Kaya ((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK) 85d1615ca2SSinan Kaya 86d1615ca2SSinan Kaya #define HIDMA_ERR_INT_MASK \ 87d1615ca2SSinan Kaya (BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS) | \ 88d1615ca2SSinan Kaya BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ 89d1615ca2SSinan Kaya BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ 90d1615ca2SSinan Kaya BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ 91d1615ca2SSinan Kaya BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS)) 92d1615ca2SSinan Kaya 93d1615ca2SSinan Kaya enum ch_command { 94d1615ca2SSinan Kaya HIDMA_CH_DISABLE = 0, 95d1615ca2SSinan Kaya HIDMA_CH_ENABLE = 1, 96d1615ca2SSinan Kaya HIDMA_CH_SUSPEND = 2, 97d1615ca2SSinan Kaya HIDMA_CH_RESET = 9, 98d1615ca2SSinan Kaya }; 99d1615ca2SSinan Kaya 100d1615ca2SSinan Kaya enum ch_state { 101d1615ca2SSinan Kaya HIDMA_CH_DISABLED = 0, 102d1615ca2SSinan Kaya HIDMA_CH_ENABLED = 1, 103d1615ca2SSinan Kaya HIDMA_CH_RUNNING = 2, 104d1615ca2SSinan Kaya HIDMA_CH_SUSPENDED = 3, 105d1615ca2SSinan Kaya HIDMA_CH_STOPPED = 4, 106d1615ca2SSinan Kaya }; 107d1615ca2SSinan Kaya 108d1615ca2SSinan Kaya enum tre_type { 109d1615ca2SSinan Kaya HIDMA_TRE_MEMCPY = 3, 110d1615ca2SSinan Kaya }; 111d1615ca2SSinan Kaya 112d1615ca2SSinan Kaya enum err_code { 113d1615ca2SSinan Kaya HIDMA_EVRE_STATUS_COMPLETE = 1, 114d1615ca2SSinan Kaya HIDMA_EVRE_STATUS_ERROR = 4, 115d1615ca2SSinan Kaya }; 116d1615ca2SSinan Kaya 117d1615ca2SSinan Kaya static int hidma_is_chan_enabled(int state) 118d1615ca2SSinan Kaya { 119d1615ca2SSinan Kaya switch (state) { 120d1615ca2SSinan Kaya case HIDMA_CH_ENABLED: 121d1615ca2SSinan Kaya case HIDMA_CH_RUNNING: 122d1615ca2SSinan Kaya return true; 123d1615ca2SSinan Kaya default: 124d1615ca2SSinan Kaya return false; 125d1615ca2SSinan Kaya } 126d1615ca2SSinan Kaya } 127d1615ca2SSinan Kaya 128d1615ca2SSinan Kaya void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch) 129d1615ca2SSinan Kaya { 130d1615ca2SSinan Kaya struct hidma_tre *tre; 131d1615ca2SSinan Kaya 132d1615ca2SSinan Kaya if (tre_ch >= lldev->nr_tres) { 133d1615ca2SSinan Kaya dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch); 134d1615ca2SSinan Kaya return; 135d1615ca2SSinan Kaya } 136d1615ca2SSinan Kaya 137d1615ca2SSinan Kaya tre = &lldev->trepool[tre_ch]; 138d1615ca2SSinan Kaya if (atomic_read(&tre->allocated) != true) { 139d1615ca2SSinan Kaya dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch); 140d1615ca2SSinan Kaya return; 141d1615ca2SSinan Kaya } 142d1615ca2SSinan Kaya 143d1615ca2SSinan Kaya atomic_set(&tre->allocated, 0); 144d1615ca2SSinan Kaya } 145d1615ca2SSinan Kaya 146d1615ca2SSinan Kaya int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name, 147d1615ca2SSinan Kaya void (*callback)(void *data), void *data, u32 *tre_ch) 148d1615ca2SSinan Kaya { 149d1615ca2SSinan Kaya unsigned int i; 150d1615ca2SSinan Kaya struct hidma_tre *tre; 151d1615ca2SSinan Kaya u32 *tre_local; 152d1615ca2SSinan Kaya 153d1615ca2SSinan Kaya if (!tre_ch || !lldev) 154d1615ca2SSinan Kaya return -EINVAL; 155d1615ca2SSinan Kaya 156d1615ca2SSinan Kaya /* need to have at least one empty spot in the queue */ 157d1615ca2SSinan Kaya for (i = 0; i < lldev->nr_tres - 1; i++) { 158d1615ca2SSinan Kaya if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1)) 159d1615ca2SSinan Kaya break; 160d1615ca2SSinan Kaya } 161d1615ca2SSinan Kaya 162d1615ca2SSinan Kaya if (i == (lldev->nr_tres - 1)) 163d1615ca2SSinan Kaya return -ENOMEM; 164d1615ca2SSinan Kaya 165d1615ca2SSinan Kaya tre = &lldev->trepool[i]; 166d1615ca2SSinan Kaya tre->dma_sig = sig; 167d1615ca2SSinan Kaya tre->dev_name = dev_name; 168d1615ca2SSinan Kaya tre->callback = callback; 169d1615ca2SSinan Kaya tre->data = data; 170d1615ca2SSinan Kaya tre->idx = i; 171d1615ca2SSinan Kaya tre->status = 0; 172d1615ca2SSinan Kaya tre->queued = 0; 173d1615ca2SSinan Kaya tre->err_code = 0; 174d1615ca2SSinan Kaya tre->err_info = 0; 175d1615ca2SSinan Kaya tre->lldev = lldev; 176d1615ca2SSinan Kaya tre_local = &tre->tre_local[0]; 177d1615ca2SSinan Kaya tre_local[HIDMA_TRE_CFG_IDX] = HIDMA_TRE_MEMCPY; 178d1615ca2SSinan Kaya tre_local[HIDMA_TRE_CFG_IDX] |= (lldev->chidx & 0xFF) << 8; 179d1615ca2SSinan Kaya tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16); /* set IEOB */ 180d1615ca2SSinan Kaya *tre_ch = i; 181d1615ca2SSinan Kaya if (callback) 182d1615ca2SSinan Kaya callback(data); 183d1615ca2SSinan Kaya return 0; 184d1615ca2SSinan Kaya } 185d1615ca2SSinan Kaya 186d1615ca2SSinan Kaya /* 187d1615ca2SSinan Kaya * Multiple TREs may be queued and waiting in the pending queue. 188d1615ca2SSinan Kaya */ 189d1615ca2SSinan Kaya static void hidma_ll_tre_complete(unsigned long arg) 190d1615ca2SSinan Kaya { 191d1615ca2SSinan Kaya struct hidma_lldev *lldev = (struct hidma_lldev *)arg; 192d1615ca2SSinan Kaya struct hidma_tre *tre; 193d1615ca2SSinan Kaya 194d1615ca2SSinan Kaya while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) { 195d1615ca2SSinan Kaya /* call the user if it has been read by the hardware */ 196d1615ca2SSinan Kaya if (tre->callback) 197d1615ca2SSinan Kaya tre->callback(tre->data); 198d1615ca2SSinan Kaya } 199d1615ca2SSinan Kaya } 200d1615ca2SSinan Kaya 201d1615ca2SSinan Kaya static int hidma_post_completed(struct hidma_lldev *lldev, int tre_iterator, 202d1615ca2SSinan Kaya u8 err_info, u8 err_code) 203d1615ca2SSinan Kaya { 204d1615ca2SSinan Kaya struct hidma_tre *tre; 205d1615ca2SSinan Kaya unsigned long flags; 206d1615ca2SSinan Kaya 207d1615ca2SSinan Kaya spin_lock_irqsave(&lldev->lock, flags); 208d1615ca2SSinan Kaya tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE]; 209d1615ca2SSinan Kaya if (!tre) { 210d1615ca2SSinan Kaya spin_unlock_irqrestore(&lldev->lock, flags); 211d1615ca2SSinan Kaya dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n", 212d1615ca2SSinan Kaya tre_iterator / HIDMA_TRE_SIZE); 213d1615ca2SSinan Kaya return -EINVAL; 214d1615ca2SSinan Kaya } 215d1615ca2SSinan Kaya lldev->pending_tre_list[tre->tre_index] = NULL; 216d1615ca2SSinan Kaya 217d1615ca2SSinan Kaya /* 218d1615ca2SSinan Kaya * Keep track of pending TREs that SW is expecting to receive 219d1615ca2SSinan Kaya * from HW. We got one now. Decrement our counter. 220d1615ca2SSinan Kaya */ 221d1615ca2SSinan Kaya lldev->pending_tre_count--; 222d1615ca2SSinan Kaya if (lldev->pending_tre_count < 0) { 223d1615ca2SSinan Kaya dev_warn(lldev->dev, "tre count mismatch on completion"); 224d1615ca2SSinan Kaya lldev->pending_tre_count = 0; 225d1615ca2SSinan Kaya } 226d1615ca2SSinan Kaya 227d1615ca2SSinan Kaya spin_unlock_irqrestore(&lldev->lock, flags); 228d1615ca2SSinan Kaya 229d1615ca2SSinan Kaya tre->err_info = err_info; 230d1615ca2SSinan Kaya tre->err_code = err_code; 231d1615ca2SSinan Kaya tre->queued = 0; 232d1615ca2SSinan Kaya 233d1615ca2SSinan Kaya kfifo_put(&lldev->handoff_fifo, tre); 234d1615ca2SSinan Kaya tasklet_schedule(&lldev->task); 235d1615ca2SSinan Kaya 236d1615ca2SSinan Kaya return 0; 237d1615ca2SSinan Kaya } 238d1615ca2SSinan Kaya 239d1615ca2SSinan Kaya /* 240d1615ca2SSinan Kaya * Called to handle the interrupt for the channel. 241d1615ca2SSinan Kaya * Return a positive number if TRE or EVRE were consumed on this run. 242d1615ca2SSinan Kaya * Return a positive number if there are pending TREs or EVREs. 243d1615ca2SSinan Kaya * Return 0 if there is nothing to consume or no pending TREs/EVREs found. 244d1615ca2SSinan Kaya */ 245d1615ca2SSinan Kaya static int hidma_handle_tre_completion(struct hidma_lldev *lldev) 246d1615ca2SSinan Kaya { 247d1615ca2SSinan Kaya u32 evre_ring_size = lldev->evre_ring_size; 248d1615ca2SSinan Kaya u32 tre_ring_size = lldev->tre_ring_size; 249d1615ca2SSinan Kaya u32 err_info, err_code, evre_write_off; 250d1615ca2SSinan Kaya u32 tre_iterator, evre_iterator; 251d1615ca2SSinan Kaya u32 num_completed = 0; 252d1615ca2SSinan Kaya 253d1615ca2SSinan Kaya evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); 254d1615ca2SSinan Kaya tre_iterator = lldev->tre_processed_off; 255d1615ca2SSinan Kaya evre_iterator = lldev->evre_processed_off; 256d1615ca2SSinan Kaya 257d1615ca2SSinan Kaya if ((evre_write_off > evre_ring_size) || 258d1615ca2SSinan Kaya (evre_write_off % HIDMA_EVRE_SIZE)) { 259d1615ca2SSinan Kaya dev_err(lldev->dev, "HW reports invalid EVRE write offset\n"); 260d1615ca2SSinan Kaya return 0; 261d1615ca2SSinan Kaya } 262d1615ca2SSinan Kaya 263d1615ca2SSinan Kaya /* 264d1615ca2SSinan Kaya * By the time control reaches here the number of EVREs and TREs 265d1615ca2SSinan Kaya * may not match. Only consume the ones that hardware told us. 266d1615ca2SSinan Kaya */ 267d1615ca2SSinan Kaya while ((evre_iterator != evre_write_off)) { 268d1615ca2SSinan Kaya u32 *current_evre = lldev->evre_ring + evre_iterator; 269d1615ca2SSinan Kaya u32 cfg; 270d1615ca2SSinan Kaya 271d1615ca2SSinan Kaya cfg = current_evre[HIDMA_EVRE_CFG_IDX]; 272d1615ca2SSinan Kaya err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS; 273d1615ca2SSinan Kaya err_info &= HIDMA_EVRE_ERRINFO_MASK; 274d1615ca2SSinan Kaya err_code = 275d1615ca2SSinan Kaya (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK; 276d1615ca2SSinan Kaya 277d1615ca2SSinan Kaya if (hidma_post_completed(lldev, tre_iterator, err_info, 278d1615ca2SSinan Kaya err_code)) 279d1615ca2SSinan Kaya break; 280d1615ca2SSinan Kaya 281d1615ca2SSinan Kaya HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, 282d1615ca2SSinan Kaya tre_ring_size); 283d1615ca2SSinan Kaya HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE, 284d1615ca2SSinan Kaya evre_ring_size); 285d1615ca2SSinan Kaya 286d1615ca2SSinan Kaya /* 287d1615ca2SSinan Kaya * Read the new event descriptor written by the HW. 288d1615ca2SSinan Kaya * As we are processing the delivered events, other events 289d1615ca2SSinan Kaya * get queued to the SW for processing. 290d1615ca2SSinan Kaya */ 291d1615ca2SSinan Kaya evre_write_off = 292d1615ca2SSinan Kaya readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); 293d1615ca2SSinan Kaya num_completed++; 294d1615ca2SSinan Kaya } 295d1615ca2SSinan Kaya 296d1615ca2SSinan Kaya if (num_completed) { 297d1615ca2SSinan Kaya u32 evre_read_off = (lldev->evre_processed_off + 298d1615ca2SSinan Kaya HIDMA_EVRE_SIZE * num_completed); 299d1615ca2SSinan Kaya u32 tre_read_off = (lldev->tre_processed_off + 300d1615ca2SSinan Kaya HIDMA_TRE_SIZE * num_completed); 301d1615ca2SSinan Kaya 302d1615ca2SSinan Kaya evre_read_off = evre_read_off % evre_ring_size; 303d1615ca2SSinan Kaya tre_read_off = tre_read_off % tre_ring_size; 304d1615ca2SSinan Kaya 305d1615ca2SSinan Kaya writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG); 306d1615ca2SSinan Kaya 307d1615ca2SSinan Kaya /* record the last processed tre offset */ 308d1615ca2SSinan Kaya lldev->tre_processed_off = tre_read_off; 309d1615ca2SSinan Kaya lldev->evre_processed_off = evre_read_off; 310d1615ca2SSinan Kaya } 311d1615ca2SSinan Kaya 312d1615ca2SSinan Kaya return num_completed; 313d1615ca2SSinan Kaya } 314d1615ca2SSinan Kaya 315d1615ca2SSinan Kaya void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info, 316d1615ca2SSinan Kaya u8 err_code) 317d1615ca2SSinan Kaya { 318d1615ca2SSinan Kaya u32 tre_iterator; 319d1615ca2SSinan Kaya u32 tre_ring_size = lldev->tre_ring_size; 320d1615ca2SSinan Kaya int num_completed = 0; 321d1615ca2SSinan Kaya u32 tre_read_off; 322d1615ca2SSinan Kaya 323d1615ca2SSinan Kaya tre_iterator = lldev->tre_processed_off; 324d1615ca2SSinan Kaya while (lldev->pending_tre_count) { 325d1615ca2SSinan Kaya if (hidma_post_completed(lldev, tre_iterator, err_info, 326d1615ca2SSinan Kaya err_code)) 327d1615ca2SSinan Kaya break; 328d1615ca2SSinan Kaya HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, 329d1615ca2SSinan Kaya tre_ring_size); 330d1615ca2SSinan Kaya num_completed++; 331d1615ca2SSinan Kaya } 332d1615ca2SSinan Kaya tre_read_off = (lldev->tre_processed_off + 333d1615ca2SSinan Kaya HIDMA_TRE_SIZE * num_completed); 334d1615ca2SSinan Kaya 335d1615ca2SSinan Kaya tre_read_off = tre_read_off % tre_ring_size; 336d1615ca2SSinan Kaya 337d1615ca2SSinan Kaya /* record the last processed tre offset */ 338d1615ca2SSinan Kaya lldev->tre_processed_off = tre_read_off; 339d1615ca2SSinan Kaya } 340d1615ca2SSinan Kaya 341d1615ca2SSinan Kaya static int hidma_ll_reset(struct hidma_lldev *lldev) 342d1615ca2SSinan Kaya { 343d1615ca2SSinan Kaya u32 val; 344d1615ca2SSinan Kaya int ret; 345d1615ca2SSinan Kaya 346d1615ca2SSinan Kaya val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 347d1615ca2SSinan Kaya val &= ~(HIDMA_CH_CONTROL_MASK << 16); 348d1615ca2SSinan Kaya val |= HIDMA_CH_RESET << 16; 349d1615ca2SSinan Kaya writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 350d1615ca2SSinan Kaya 351d1615ca2SSinan Kaya /* 352d1615ca2SSinan Kaya * Delay 10ms after reset to allow DMA logic to quiesce. 353d1615ca2SSinan Kaya * Do a polled read up to 1ms and 10ms maximum. 354d1615ca2SSinan Kaya */ 355d1615ca2SSinan Kaya ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, 356d1615ca2SSinan Kaya HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, 357d1615ca2SSinan Kaya 1000, 10000); 358d1615ca2SSinan Kaya if (ret) { 359d1615ca2SSinan Kaya dev_err(lldev->dev, "transfer channel did not reset\n"); 360d1615ca2SSinan Kaya return ret; 361d1615ca2SSinan Kaya } 362d1615ca2SSinan Kaya 363d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 364d1615ca2SSinan Kaya val &= ~(HIDMA_CH_CONTROL_MASK << 16); 365d1615ca2SSinan Kaya val |= HIDMA_CH_RESET << 16; 366d1615ca2SSinan Kaya writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 367d1615ca2SSinan Kaya 368d1615ca2SSinan Kaya /* 369d1615ca2SSinan Kaya * Delay 10ms after reset to allow DMA logic to quiesce. 370d1615ca2SSinan Kaya * Do a polled read up to 1ms and 10ms maximum. 371d1615ca2SSinan Kaya */ 372d1615ca2SSinan Kaya ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, 373d1615ca2SSinan Kaya HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, 374d1615ca2SSinan Kaya 1000, 10000); 375d1615ca2SSinan Kaya if (ret) 376d1615ca2SSinan Kaya return ret; 377d1615ca2SSinan Kaya 378d1615ca2SSinan Kaya lldev->trch_state = HIDMA_CH_DISABLED; 379d1615ca2SSinan Kaya lldev->evch_state = HIDMA_CH_DISABLED; 380d1615ca2SSinan Kaya return 0; 381d1615ca2SSinan Kaya } 382d1615ca2SSinan Kaya 383d1615ca2SSinan Kaya /* 384d1615ca2SSinan Kaya * Abort all transactions and perform a reset. 385d1615ca2SSinan Kaya */ 386d1615ca2SSinan Kaya static void hidma_ll_abort(unsigned long arg) 387d1615ca2SSinan Kaya { 388d1615ca2SSinan Kaya struct hidma_lldev *lldev = (struct hidma_lldev *)arg; 389d1615ca2SSinan Kaya u8 err_code = HIDMA_EVRE_STATUS_ERROR; 390d1615ca2SSinan Kaya u8 err_info = 0xFF; 391d1615ca2SSinan Kaya int rc; 392d1615ca2SSinan Kaya 393d1615ca2SSinan Kaya hidma_cleanup_pending_tre(lldev, err_info, err_code); 394d1615ca2SSinan Kaya 395d1615ca2SSinan Kaya /* reset the channel for recovery */ 396d1615ca2SSinan Kaya rc = hidma_ll_setup(lldev); 397d1615ca2SSinan Kaya if (rc) { 398d1615ca2SSinan Kaya dev_err(lldev->dev, "channel reinitialize failed after error\n"); 399d1615ca2SSinan Kaya return; 400d1615ca2SSinan Kaya } 401d1615ca2SSinan Kaya writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 402d1615ca2SSinan Kaya } 403d1615ca2SSinan Kaya 404d1615ca2SSinan Kaya /* 405d1615ca2SSinan Kaya * The interrupt handler for HIDMA will try to consume as many pending 406d1615ca2SSinan Kaya * EVRE from the event queue as possible. Each EVRE has an associated 407d1615ca2SSinan Kaya * TRE that holds the user interface parameters. EVRE reports the 408d1615ca2SSinan Kaya * result of the transaction. Hardware guarantees ordering between EVREs 409d1615ca2SSinan Kaya * and TREs. We use last processed offset to figure out which TRE is 410d1615ca2SSinan Kaya * associated with which EVRE. If two TREs are consumed by HW, the EVREs 411d1615ca2SSinan Kaya * are in order in the event ring. 412d1615ca2SSinan Kaya * 413d1615ca2SSinan Kaya * This handler will do a one pass for consuming EVREs. Other EVREs may 414d1615ca2SSinan Kaya * be delivered while we are working. It will try to consume incoming 415d1615ca2SSinan Kaya * EVREs one more time and return. 416d1615ca2SSinan Kaya * 417d1615ca2SSinan Kaya * For unprocessed EVREs, hardware will trigger another interrupt until 418d1615ca2SSinan Kaya * all the interrupt bits are cleared. 419d1615ca2SSinan Kaya * 420d1615ca2SSinan Kaya * Hardware guarantees that by the time interrupt is observed, all data 421d1615ca2SSinan Kaya * transactions in flight are delivered to their respective places and 422d1615ca2SSinan Kaya * are visible to the CPU. 423d1615ca2SSinan Kaya * 424d1615ca2SSinan Kaya * On demand paging for IOMMU is only supported for PCIe via PRI 425d1615ca2SSinan Kaya * (Page Request Interface) not for HIDMA. All other hardware instances 426d1615ca2SSinan Kaya * including HIDMA work on pinned DMA addresses. 427d1615ca2SSinan Kaya * 428d1615ca2SSinan Kaya * HIDMA is not aware of IOMMU presence since it follows the DMA API. All 429d1615ca2SSinan Kaya * IOMMU latency will be built into the data movement time. By the time 430d1615ca2SSinan Kaya * interrupt happens, IOMMU lookups + data movement has already taken place. 431d1615ca2SSinan Kaya * 432d1615ca2SSinan Kaya * While the first read in a typical PCI endpoint ISR flushes all outstanding 433d1615ca2SSinan Kaya * requests traditionally to the destination, this concept does not apply 434d1615ca2SSinan Kaya * here for this HW. 435d1615ca2SSinan Kaya */ 436d1615ca2SSinan Kaya irqreturn_t hidma_ll_inthandler(int chirq, void *arg) 437d1615ca2SSinan Kaya { 438d1615ca2SSinan Kaya struct hidma_lldev *lldev = arg; 439d1615ca2SSinan Kaya u32 status; 440d1615ca2SSinan Kaya u32 enable; 441d1615ca2SSinan Kaya u32 cause; 442d1615ca2SSinan Kaya 443d1615ca2SSinan Kaya /* 444d1615ca2SSinan Kaya * Fine tuned for this HW... 445d1615ca2SSinan Kaya * 446d1615ca2SSinan Kaya * This ISR has been designed for this particular hardware. Relaxed 447d1615ca2SSinan Kaya * read and write accessors are used for performance reasons due to 448d1615ca2SSinan Kaya * interrupt delivery guarantees. Do not copy this code blindly and 449d1615ca2SSinan Kaya * expect that to work. 450d1615ca2SSinan Kaya */ 451d1615ca2SSinan Kaya status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 452d1615ca2SSinan Kaya enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 453d1615ca2SSinan Kaya cause = status & enable; 454d1615ca2SSinan Kaya 455d1615ca2SSinan Kaya while (cause) { 456d1615ca2SSinan Kaya if (cause & HIDMA_ERR_INT_MASK) { 457d1615ca2SSinan Kaya dev_err(lldev->dev, "error 0x%x, resetting...\n", 458d1615ca2SSinan Kaya cause); 459d1615ca2SSinan Kaya 460d1615ca2SSinan Kaya /* Clear out pending interrupts */ 461d1615ca2SSinan Kaya writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 462d1615ca2SSinan Kaya 463d1615ca2SSinan Kaya tasklet_schedule(&lldev->rst_task); 464d1615ca2SSinan Kaya goto out; 465d1615ca2SSinan Kaya } 466d1615ca2SSinan Kaya 467d1615ca2SSinan Kaya /* 468d1615ca2SSinan Kaya * Try to consume as many EVREs as possible. 469d1615ca2SSinan Kaya */ 470d1615ca2SSinan Kaya hidma_handle_tre_completion(lldev); 471d1615ca2SSinan Kaya 472d1615ca2SSinan Kaya /* We consumed TREs or there are pending TREs or EVREs. */ 473d1615ca2SSinan Kaya writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 474d1615ca2SSinan Kaya 475d1615ca2SSinan Kaya /* 476d1615ca2SSinan Kaya * Another interrupt might have arrived while we are 477d1615ca2SSinan Kaya * processing this one. Read the new cause. 478d1615ca2SSinan Kaya */ 479d1615ca2SSinan Kaya status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 480d1615ca2SSinan Kaya enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 481d1615ca2SSinan Kaya cause = status & enable; 482d1615ca2SSinan Kaya } 483d1615ca2SSinan Kaya 484d1615ca2SSinan Kaya out: 485d1615ca2SSinan Kaya return IRQ_HANDLED; 486d1615ca2SSinan Kaya } 487d1615ca2SSinan Kaya 488d1615ca2SSinan Kaya int hidma_ll_enable(struct hidma_lldev *lldev) 489d1615ca2SSinan Kaya { 490d1615ca2SSinan Kaya u32 val; 491d1615ca2SSinan Kaya int ret; 492d1615ca2SSinan Kaya 493d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 494d1615ca2SSinan Kaya val &= ~(HIDMA_CH_CONTROL_MASK << 16); 495d1615ca2SSinan Kaya val |= HIDMA_CH_ENABLE << 16; 496d1615ca2SSinan Kaya writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 497d1615ca2SSinan Kaya 498d1615ca2SSinan Kaya ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, 499d1615ca2SSinan Kaya hidma_is_chan_enabled(HIDMA_CH_STATE(val)), 500d1615ca2SSinan Kaya 1000, 10000); 501d1615ca2SSinan Kaya if (ret) { 502d1615ca2SSinan Kaya dev_err(lldev->dev, "event channel did not get enabled\n"); 503d1615ca2SSinan Kaya return ret; 504d1615ca2SSinan Kaya } 505d1615ca2SSinan Kaya 506d1615ca2SSinan Kaya val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 507d1615ca2SSinan Kaya val &= ~(HIDMA_CH_CONTROL_MASK << 16); 508d1615ca2SSinan Kaya val |= HIDMA_CH_ENABLE << 16; 509d1615ca2SSinan Kaya writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 510d1615ca2SSinan Kaya 511d1615ca2SSinan Kaya ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, 512d1615ca2SSinan Kaya hidma_is_chan_enabled(HIDMA_CH_STATE(val)), 513d1615ca2SSinan Kaya 1000, 10000); 514d1615ca2SSinan Kaya if (ret) { 515d1615ca2SSinan Kaya dev_err(lldev->dev, "transfer channel did not get enabled\n"); 516d1615ca2SSinan Kaya return ret; 517d1615ca2SSinan Kaya } 518d1615ca2SSinan Kaya 519d1615ca2SSinan Kaya lldev->trch_state = HIDMA_CH_ENABLED; 520d1615ca2SSinan Kaya lldev->evch_state = HIDMA_CH_ENABLED; 521d1615ca2SSinan Kaya 522d1615ca2SSinan Kaya return 0; 523d1615ca2SSinan Kaya } 524d1615ca2SSinan Kaya 525d1615ca2SSinan Kaya void hidma_ll_start(struct hidma_lldev *lldev) 526d1615ca2SSinan Kaya { 527d1615ca2SSinan Kaya unsigned long irqflags; 528d1615ca2SSinan Kaya 529d1615ca2SSinan Kaya spin_lock_irqsave(&lldev->lock, irqflags); 530d1615ca2SSinan Kaya writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG); 531d1615ca2SSinan Kaya spin_unlock_irqrestore(&lldev->lock, irqflags); 532d1615ca2SSinan Kaya } 533d1615ca2SSinan Kaya 534d1615ca2SSinan Kaya bool hidma_ll_isenabled(struct hidma_lldev *lldev) 535d1615ca2SSinan Kaya { 536d1615ca2SSinan Kaya u32 val; 537d1615ca2SSinan Kaya 538d1615ca2SSinan Kaya val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 539d1615ca2SSinan Kaya lldev->trch_state = HIDMA_CH_STATE(val); 540d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 541d1615ca2SSinan Kaya lldev->evch_state = HIDMA_CH_STATE(val); 542d1615ca2SSinan Kaya 543d1615ca2SSinan Kaya /* both channels have to be enabled before calling this function */ 544d1615ca2SSinan Kaya if (hidma_is_chan_enabled(lldev->trch_state) && 545d1615ca2SSinan Kaya hidma_is_chan_enabled(lldev->evch_state)) 546d1615ca2SSinan Kaya return true; 547d1615ca2SSinan Kaya 548d1615ca2SSinan Kaya return false; 549d1615ca2SSinan Kaya } 550d1615ca2SSinan Kaya 551d1615ca2SSinan Kaya void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch) 552d1615ca2SSinan Kaya { 553d1615ca2SSinan Kaya struct hidma_tre *tre; 554d1615ca2SSinan Kaya unsigned long flags; 555d1615ca2SSinan Kaya 556d1615ca2SSinan Kaya tre = &lldev->trepool[tre_ch]; 557d1615ca2SSinan Kaya 558d1615ca2SSinan Kaya /* copy the TRE into its location in the TRE ring */ 559d1615ca2SSinan Kaya spin_lock_irqsave(&lldev->lock, flags); 560d1615ca2SSinan Kaya tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE; 561d1615ca2SSinan Kaya lldev->pending_tre_list[tre->tre_index] = tre; 562d1615ca2SSinan Kaya memcpy(lldev->tre_ring + lldev->tre_write_offset, 563d1615ca2SSinan Kaya &tre->tre_local[0], HIDMA_TRE_SIZE); 564d1615ca2SSinan Kaya tre->err_code = 0; 565d1615ca2SSinan Kaya tre->err_info = 0; 566d1615ca2SSinan Kaya tre->queued = 1; 567d1615ca2SSinan Kaya lldev->pending_tre_count++; 568d1615ca2SSinan Kaya lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE) 569d1615ca2SSinan Kaya % lldev->tre_ring_size; 570d1615ca2SSinan Kaya spin_unlock_irqrestore(&lldev->lock, flags); 571d1615ca2SSinan Kaya } 572d1615ca2SSinan Kaya 573d1615ca2SSinan Kaya /* 574d1615ca2SSinan Kaya * Note that even though we stop this channel if there is a pending transaction 575d1615ca2SSinan Kaya * in flight it will complete and follow the callback. This request will 576d1615ca2SSinan Kaya * prevent further requests to be made. 577d1615ca2SSinan Kaya */ 578d1615ca2SSinan Kaya int hidma_ll_disable(struct hidma_lldev *lldev) 579d1615ca2SSinan Kaya { 580d1615ca2SSinan Kaya u32 val; 581d1615ca2SSinan Kaya int ret; 582d1615ca2SSinan Kaya 583d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 584d1615ca2SSinan Kaya lldev->evch_state = HIDMA_CH_STATE(val); 585d1615ca2SSinan Kaya val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 586d1615ca2SSinan Kaya lldev->trch_state = HIDMA_CH_STATE(val); 587d1615ca2SSinan Kaya 588d1615ca2SSinan Kaya /* already suspended by this OS */ 589d1615ca2SSinan Kaya if ((lldev->trch_state == HIDMA_CH_SUSPENDED) || 590d1615ca2SSinan Kaya (lldev->evch_state == HIDMA_CH_SUSPENDED)) 591d1615ca2SSinan Kaya return 0; 592d1615ca2SSinan Kaya 593d1615ca2SSinan Kaya /* already stopped by the manager */ 594d1615ca2SSinan Kaya if ((lldev->trch_state == HIDMA_CH_STOPPED) || 595d1615ca2SSinan Kaya (lldev->evch_state == HIDMA_CH_STOPPED)) 596d1615ca2SSinan Kaya return 0; 597d1615ca2SSinan Kaya 598d1615ca2SSinan Kaya val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 599d1615ca2SSinan Kaya val &= ~(HIDMA_CH_CONTROL_MASK << 16); 600d1615ca2SSinan Kaya val |= HIDMA_CH_SUSPEND << 16; 601d1615ca2SSinan Kaya writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); 602d1615ca2SSinan Kaya 603d1615ca2SSinan Kaya /* 604d1615ca2SSinan Kaya * Start the wait right after the suspend is confirmed. 605d1615ca2SSinan Kaya * Do a polled read up to 1ms and 10ms maximum. 606d1615ca2SSinan Kaya */ 607d1615ca2SSinan Kaya ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, 608d1615ca2SSinan Kaya HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, 609d1615ca2SSinan Kaya 1000, 10000); 610d1615ca2SSinan Kaya if (ret) 611d1615ca2SSinan Kaya return ret; 612d1615ca2SSinan Kaya 613d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 614d1615ca2SSinan Kaya val &= ~(HIDMA_CH_CONTROL_MASK << 16); 615d1615ca2SSinan Kaya val |= HIDMA_CH_SUSPEND << 16; 616d1615ca2SSinan Kaya writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); 617d1615ca2SSinan Kaya 618d1615ca2SSinan Kaya /* 619d1615ca2SSinan Kaya * Start the wait right after the suspend is confirmed 620d1615ca2SSinan Kaya * Delay up to 10ms after reset to allow DMA logic to quiesce. 621d1615ca2SSinan Kaya */ 622d1615ca2SSinan Kaya ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, 623d1615ca2SSinan Kaya HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, 624d1615ca2SSinan Kaya 1000, 10000); 625d1615ca2SSinan Kaya if (ret) 626d1615ca2SSinan Kaya return ret; 627d1615ca2SSinan Kaya 628d1615ca2SSinan Kaya lldev->trch_state = HIDMA_CH_SUSPENDED; 629d1615ca2SSinan Kaya lldev->evch_state = HIDMA_CH_SUSPENDED; 630d1615ca2SSinan Kaya return 0; 631d1615ca2SSinan Kaya } 632d1615ca2SSinan Kaya 633d1615ca2SSinan Kaya void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch, 634d1615ca2SSinan Kaya dma_addr_t src, dma_addr_t dest, u32 len, 635d1615ca2SSinan Kaya u32 flags) 636d1615ca2SSinan Kaya { 637d1615ca2SSinan Kaya struct hidma_tre *tre; 638d1615ca2SSinan Kaya u32 *tre_local; 639d1615ca2SSinan Kaya 640d1615ca2SSinan Kaya if (tre_ch >= lldev->nr_tres) { 641d1615ca2SSinan Kaya dev_err(lldev->dev, "invalid TRE number in transfer params:%d", 642d1615ca2SSinan Kaya tre_ch); 643d1615ca2SSinan Kaya return; 644d1615ca2SSinan Kaya } 645d1615ca2SSinan Kaya 646d1615ca2SSinan Kaya tre = &lldev->trepool[tre_ch]; 647d1615ca2SSinan Kaya if (atomic_read(&tre->allocated) != true) { 648d1615ca2SSinan Kaya dev_err(lldev->dev, "trying to set params on an unused TRE:%d", 649d1615ca2SSinan Kaya tre_ch); 650d1615ca2SSinan Kaya return; 651d1615ca2SSinan Kaya } 652d1615ca2SSinan Kaya 653d1615ca2SSinan Kaya tre_local = &tre->tre_local[0]; 654d1615ca2SSinan Kaya tre_local[HIDMA_TRE_LEN_IDX] = len; 655d1615ca2SSinan Kaya tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src); 656d1615ca2SSinan Kaya tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src); 657d1615ca2SSinan Kaya tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest); 658d1615ca2SSinan Kaya tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest); 659d1615ca2SSinan Kaya tre->int_flags = flags; 660d1615ca2SSinan Kaya } 661d1615ca2SSinan Kaya 662d1615ca2SSinan Kaya /* 663d1615ca2SSinan Kaya * Called during initialization and after an error condition 664d1615ca2SSinan Kaya * to restore hardware state. 665d1615ca2SSinan Kaya */ 666d1615ca2SSinan Kaya int hidma_ll_setup(struct hidma_lldev *lldev) 667d1615ca2SSinan Kaya { 668d1615ca2SSinan Kaya int rc; 669d1615ca2SSinan Kaya u64 addr; 670d1615ca2SSinan Kaya u32 val; 671d1615ca2SSinan Kaya u32 nr_tres = lldev->nr_tres; 672d1615ca2SSinan Kaya 673d1615ca2SSinan Kaya lldev->pending_tre_count = 0; 674d1615ca2SSinan Kaya lldev->tre_processed_off = 0; 675d1615ca2SSinan Kaya lldev->evre_processed_off = 0; 676d1615ca2SSinan Kaya lldev->tre_write_offset = 0; 677d1615ca2SSinan Kaya 678d1615ca2SSinan Kaya /* disable interrupts */ 679d1615ca2SSinan Kaya writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 680d1615ca2SSinan Kaya 681d1615ca2SSinan Kaya /* clear all pending interrupts */ 682d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 683d1615ca2SSinan Kaya writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 684d1615ca2SSinan Kaya 685d1615ca2SSinan Kaya rc = hidma_ll_reset(lldev); 686d1615ca2SSinan Kaya if (rc) 687d1615ca2SSinan Kaya return rc; 688d1615ca2SSinan Kaya 689d1615ca2SSinan Kaya /* 690d1615ca2SSinan Kaya * Clear all pending interrupts again. 691d1615ca2SSinan Kaya * Otherwise, we observe reset complete interrupts. 692d1615ca2SSinan Kaya */ 693d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 694d1615ca2SSinan Kaya writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 695d1615ca2SSinan Kaya 696d1615ca2SSinan Kaya /* disable interrupts again after reset */ 697d1615ca2SSinan Kaya writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 698d1615ca2SSinan Kaya 699d1615ca2SSinan Kaya addr = lldev->tre_dma; 700d1615ca2SSinan Kaya writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG); 701d1615ca2SSinan Kaya writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG); 702d1615ca2SSinan Kaya writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG); 703d1615ca2SSinan Kaya 704d1615ca2SSinan Kaya addr = lldev->evre_dma; 705d1615ca2SSinan Kaya writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG); 706d1615ca2SSinan Kaya writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG); 707d1615ca2SSinan Kaya writel(HIDMA_EVRE_SIZE * nr_tres, 708d1615ca2SSinan Kaya lldev->evca + HIDMA_EVCA_RING_LEN_REG); 709d1615ca2SSinan Kaya 710d1615ca2SSinan Kaya /* support IRQ only for now */ 711d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG); 712d1615ca2SSinan Kaya val &= ~0xF; 713d1615ca2SSinan Kaya val |= 0x1; 714d1615ca2SSinan Kaya writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG); 715d1615ca2SSinan Kaya 716d1615ca2SSinan Kaya /* clear all pending interrupts and enable them */ 717d1615ca2SSinan Kaya writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 718d1615ca2SSinan Kaya writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 719d1615ca2SSinan Kaya 720d1615ca2SSinan Kaya return hidma_ll_enable(lldev); 721d1615ca2SSinan Kaya } 722d1615ca2SSinan Kaya 723d1615ca2SSinan Kaya struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres, 724d1615ca2SSinan Kaya void __iomem *trca, void __iomem *evca, 725d1615ca2SSinan Kaya u8 chidx) 726d1615ca2SSinan Kaya { 727d1615ca2SSinan Kaya u32 required_bytes; 728d1615ca2SSinan Kaya struct hidma_lldev *lldev; 729d1615ca2SSinan Kaya int rc; 730d1615ca2SSinan Kaya size_t sz; 731d1615ca2SSinan Kaya 732d1615ca2SSinan Kaya if (!trca || !evca || !dev || !nr_tres) 733d1615ca2SSinan Kaya return NULL; 734d1615ca2SSinan Kaya 735d1615ca2SSinan Kaya /* need at least four TREs */ 736d1615ca2SSinan Kaya if (nr_tres < 4) 737d1615ca2SSinan Kaya return NULL; 738d1615ca2SSinan Kaya 739d1615ca2SSinan Kaya /* need an extra space */ 740d1615ca2SSinan Kaya nr_tres += 1; 741d1615ca2SSinan Kaya 742d1615ca2SSinan Kaya lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL); 743d1615ca2SSinan Kaya if (!lldev) 744d1615ca2SSinan Kaya return NULL; 745d1615ca2SSinan Kaya 746d1615ca2SSinan Kaya lldev->evca = evca; 747d1615ca2SSinan Kaya lldev->trca = trca; 748d1615ca2SSinan Kaya lldev->dev = dev; 749d1615ca2SSinan Kaya sz = sizeof(struct hidma_tre); 750d1615ca2SSinan Kaya lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL); 751d1615ca2SSinan Kaya if (!lldev->trepool) 752d1615ca2SSinan Kaya return NULL; 753d1615ca2SSinan Kaya 754d1615ca2SSinan Kaya required_bytes = sizeof(lldev->pending_tre_list[0]); 755d1615ca2SSinan Kaya lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes, 756d1615ca2SSinan Kaya GFP_KERNEL); 757d1615ca2SSinan Kaya if (!lldev->pending_tre_list) 758d1615ca2SSinan Kaya return NULL; 759d1615ca2SSinan Kaya 760d1615ca2SSinan Kaya sz = (HIDMA_TRE_SIZE + 1) * nr_tres; 761d1615ca2SSinan Kaya lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma, 762d1615ca2SSinan Kaya GFP_KERNEL); 763d1615ca2SSinan Kaya if (!lldev->tre_ring) 764d1615ca2SSinan Kaya return NULL; 765d1615ca2SSinan Kaya 766d1615ca2SSinan Kaya memset(lldev->tre_ring, 0, (HIDMA_TRE_SIZE + 1) * nr_tres); 767d1615ca2SSinan Kaya lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres; 768d1615ca2SSinan Kaya lldev->nr_tres = nr_tres; 769d1615ca2SSinan Kaya 770d1615ca2SSinan Kaya /* the TRE ring has to be TRE_SIZE aligned */ 771d1615ca2SSinan Kaya if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) { 772d1615ca2SSinan Kaya u8 tre_ring_shift; 773d1615ca2SSinan Kaya 774d1615ca2SSinan Kaya tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE; 775d1615ca2SSinan Kaya tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift; 776d1615ca2SSinan Kaya lldev->tre_dma += tre_ring_shift; 777d1615ca2SSinan Kaya lldev->tre_ring += tre_ring_shift; 778d1615ca2SSinan Kaya } 779d1615ca2SSinan Kaya 780d1615ca2SSinan Kaya sz = (HIDMA_EVRE_SIZE + 1) * nr_tres; 781d1615ca2SSinan Kaya lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma, 782d1615ca2SSinan Kaya GFP_KERNEL); 783d1615ca2SSinan Kaya if (!lldev->evre_ring) 784d1615ca2SSinan Kaya return NULL; 785d1615ca2SSinan Kaya 786d1615ca2SSinan Kaya memset(lldev->evre_ring, 0, (HIDMA_EVRE_SIZE + 1) * nr_tres); 787d1615ca2SSinan Kaya lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres; 788d1615ca2SSinan Kaya 789d1615ca2SSinan Kaya /* the EVRE ring has to be EVRE_SIZE aligned */ 790d1615ca2SSinan Kaya if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) { 791d1615ca2SSinan Kaya u8 evre_ring_shift; 792d1615ca2SSinan Kaya 793d1615ca2SSinan Kaya evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE; 794d1615ca2SSinan Kaya evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift; 795d1615ca2SSinan Kaya lldev->evre_dma += evre_ring_shift; 796d1615ca2SSinan Kaya lldev->evre_ring += evre_ring_shift; 797d1615ca2SSinan Kaya } 798d1615ca2SSinan Kaya lldev->nr_tres = nr_tres; 799d1615ca2SSinan Kaya lldev->chidx = chidx; 800d1615ca2SSinan Kaya 801d1615ca2SSinan Kaya sz = nr_tres * sizeof(struct hidma_tre *); 802d1615ca2SSinan Kaya rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL); 803d1615ca2SSinan Kaya if (rc) 804d1615ca2SSinan Kaya return NULL; 805d1615ca2SSinan Kaya 806d1615ca2SSinan Kaya rc = hidma_ll_setup(lldev); 807d1615ca2SSinan Kaya if (rc) 808d1615ca2SSinan Kaya return NULL; 809d1615ca2SSinan Kaya 810d1615ca2SSinan Kaya spin_lock_init(&lldev->lock); 811d1615ca2SSinan Kaya tasklet_init(&lldev->rst_task, hidma_ll_abort, (unsigned long)lldev); 812d1615ca2SSinan Kaya tasklet_init(&lldev->task, hidma_ll_tre_complete, (unsigned long)lldev); 813d1615ca2SSinan Kaya lldev->initialized = 1; 814d1615ca2SSinan Kaya writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 815d1615ca2SSinan Kaya return lldev; 816d1615ca2SSinan Kaya } 817d1615ca2SSinan Kaya 818d1615ca2SSinan Kaya int hidma_ll_uninit(struct hidma_lldev *lldev) 819d1615ca2SSinan Kaya { 820d1615ca2SSinan Kaya u32 required_bytes; 821d1615ca2SSinan Kaya int rc = 0; 822d1615ca2SSinan Kaya u32 val; 823d1615ca2SSinan Kaya 824d1615ca2SSinan Kaya if (!lldev) 825d1615ca2SSinan Kaya return -ENODEV; 826d1615ca2SSinan Kaya 827d1615ca2SSinan Kaya if (!lldev->initialized) 828d1615ca2SSinan Kaya return 0; 829d1615ca2SSinan Kaya 830d1615ca2SSinan Kaya lldev->initialized = 0; 831d1615ca2SSinan Kaya 832d1615ca2SSinan Kaya required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres; 833d1615ca2SSinan Kaya tasklet_kill(&lldev->task); 834d1615ca2SSinan Kaya memset(lldev->trepool, 0, required_bytes); 835d1615ca2SSinan Kaya lldev->trepool = NULL; 836d1615ca2SSinan Kaya lldev->pending_tre_count = 0; 837d1615ca2SSinan Kaya lldev->tre_write_offset = 0; 838d1615ca2SSinan Kaya 839d1615ca2SSinan Kaya rc = hidma_ll_reset(lldev); 840d1615ca2SSinan Kaya 841d1615ca2SSinan Kaya /* 842d1615ca2SSinan Kaya * Clear all pending interrupts again. 843d1615ca2SSinan Kaya * Otherwise, we observe reset complete interrupts. 844d1615ca2SSinan Kaya */ 845d1615ca2SSinan Kaya val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); 846d1615ca2SSinan Kaya writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); 847d1615ca2SSinan Kaya writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); 848d1615ca2SSinan Kaya return rc; 849d1615ca2SSinan Kaya } 850d1615ca2SSinan Kaya 851d1615ca2SSinan Kaya enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch) 852d1615ca2SSinan Kaya { 853d1615ca2SSinan Kaya enum dma_status ret = DMA_ERROR; 854d1615ca2SSinan Kaya struct hidma_tre *tre; 855d1615ca2SSinan Kaya unsigned long flags; 856d1615ca2SSinan Kaya u8 err_code; 857d1615ca2SSinan Kaya 858d1615ca2SSinan Kaya spin_lock_irqsave(&lldev->lock, flags); 859d1615ca2SSinan Kaya 860d1615ca2SSinan Kaya tre = &lldev->trepool[tre_ch]; 861d1615ca2SSinan Kaya err_code = tre->err_code; 862d1615ca2SSinan Kaya 863d1615ca2SSinan Kaya if (err_code & HIDMA_EVRE_STATUS_COMPLETE) 864d1615ca2SSinan Kaya ret = DMA_COMPLETE; 865d1615ca2SSinan Kaya else if (err_code & HIDMA_EVRE_STATUS_ERROR) 866d1615ca2SSinan Kaya ret = DMA_ERROR; 867d1615ca2SSinan Kaya else 868d1615ca2SSinan Kaya ret = DMA_IN_PROGRESS; 869d1615ca2SSinan Kaya spin_unlock_irqrestore(&lldev->lock, flags); 870d1615ca2SSinan Kaya 871d1615ca2SSinan Kaya return ret; 872d1615ca2SSinan Kaya } 873