1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Passthru DMA device driver 4 * -- Based on the CCP driver 5 * 6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. 7 * 8 * Author: Sanjay R Mehta <sanju.mehta@amd.com> 9 * Author: Gary R Hook <gary.hook@amd.com> 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/debugfs.h> 15 #include <linux/interrupt.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/pci.h> 19 20 #include "ptdma.h" 21 22 /* Human-readable error strings */ 23 static char *pt_error_codes[] = { 24 "", 25 "ERR 01: ILLEGAL_ENGINE", 26 "ERR 03: ILLEGAL_FUNCTION_TYPE", 27 "ERR 04: ILLEGAL_FUNCTION_MODE", 28 "ERR 06: ILLEGAL_FUNCTION_SIZE", 29 "ERR 08: ILLEGAL_FUNCTION_RSVD", 30 "ERR 09: ILLEGAL_BUFFER_LENGTH", 31 "ERR 10: VLSB_FAULT", 32 "ERR 11: ILLEGAL_MEM_ADDR", 33 "ERR 12: ILLEGAL_MEM_SEL", 34 "ERR 13: ILLEGAL_CONTEXT_ID", 35 "ERR 15: 0xF Reserved", 36 "ERR 18: CMD_TIMEOUT", 37 "ERR 19: IDMA0_AXI_SLVERR", 38 "ERR 20: IDMA0_AXI_DECERR", 39 "ERR 21: 0x15 Reserved", 40 "ERR 22: IDMA1_AXI_SLAVE_FAULT", 41 "ERR 23: IDMA1_AIXI_DECERR", 42 "ERR 24: 0x18 Reserved", 43 "ERR 27: 0x1B Reserved", 44 "ERR 38: ODMA0_AXI_SLVERR", 45 "ERR 39: ODMA0_AXI_DECERR", 46 "ERR 40: 0x28 Reserved", 47 "ERR 41: ODMA1_AXI_SLVERR", 48 "ERR 42: ODMA1_AXI_DECERR", 49 "ERR 43: LSB_PARITY_ERR", 50 }; 51 52 static void pt_log_error(struct pt_device *d, int e) 53 { 54 dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e); 55 } 56 57 void pt_start_queue(struct pt_cmd_queue *cmd_q) 58 { 59 /* Turn on the run bit */ 60 iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control); 61 } 62 63 void pt_stop_queue(struct pt_cmd_queue *cmd_q) 64 { 65 /* Turn off the run bit */ 66 iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control); 67 } 68 69 static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q) 70 { 71 bool soc = FIELD_GET(DWORD0_SOC, desc->dw0); 72 u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx]; 73 u32 tail; 74 75 if (soc) { 76 desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0); 77 desc->dw0 &= ~DWORD0_SOC; 78 } 79 mutex_lock(&cmd_q->q_mutex); 80 81 /* Copy 32-byte command descriptor to hw queue. */ 82 memcpy(q_desc, desc, 32); 83 cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN; 84 85 /* The data used by this command must be flushed to memory */ 86 wmb(); 87 88 /* Write the new tail address back to the queue register */ 89 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); 90 iowrite32(tail, cmd_q->reg_control + 0x0004); 91 92 /* Turn the queue back on using our cached control register */ 93 pt_start_queue(cmd_q); 94 mutex_unlock(&cmd_q->q_mutex); 95 96 return 0; 97 } 98 99 int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, 100 struct pt_passthru_engine *pt_engine) 101 { 102 struct ptdma_desc desc; 103 104 cmd_q->cmd_error = 0; 105 cmd_q->total_pt_ops++; 106 memset(&desc, 0, sizeof(desc)); 107 desc.dw0 = CMD_DESC_DW0_VAL; 108 desc.length = pt_engine->src_len; 109 desc.src_lo = lower_32_bits(pt_engine->src_dma); 110 desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma); 111 desc.dst_lo = lower_32_bits(pt_engine->dst_dma); 112 desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma); 113 114 return pt_core_execute_cmd(&desc, cmd_q); 115 } 116 117 static inline void pt_core_disable_queue_interrupts(struct pt_device *pt) 118 { 119 iowrite32(0, pt->cmd_q.reg_control + 0x000C); 120 } 121 122 static inline void pt_core_enable_queue_interrupts(struct pt_device *pt) 123 { 124 iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C); 125 } 126 127 static void pt_do_cmd_complete(unsigned long data) 128 { 129 struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data; 130 struct pt_cmd *cmd = tdata->cmd; 131 struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q; 132 u32 tail; 133 134 if (cmd_q->cmd_error) { 135 /* 136 * Log the error and flush the queue by 137 * moving the head pointer 138 */ 139 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); 140 pt_log_error(cmd_q->pt, cmd_q->cmd_error); 141 iowrite32(tail, cmd_q->reg_control + 0x0008); 142 } 143 144 cmd->pt_cmd_callback(cmd->data, cmd->ret); 145 } 146 147 static irqreturn_t pt_core_irq_handler(int irq, void *data) 148 { 149 struct pt_device *pt = data; 150 struct pt_cmd_queue *cmd_q = &pt->cmd_q; 151 u32 status; 152 153 pt_core_disable_queue_interrupts(pt); 154 pt->total_interrupts++; 155 status = ioread32(cmd_q->reg_control + 0x0010); 156 if (status) { 157 cmd_q->int_status = status; 158 cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100); 159 cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104); 160 161 /* On error, only save the first error value */ 162 if ((status & INT_ERROR) && !cmd_q->cmd_error) 163 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 164 165 /* Acknowledge the interrupt */ 166 iowrite32(status, cmd_q->reg_control + 0x0010); 167 pt_core_enable_queue_interrupts(pt); 168 pt_do_cmd_complete((ulong)&pt->tdata); 169 } 170 return IRQ_HANDLED; 171 } 172 173 int pt_core_init(struct pt_device *pt) 174 { 175 char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 176 struct pt_cmd_queue *cmd_q = &pt->cmd_q; 177 u32 dma_addr_lo, dma_addr_hi; 178 struct device *dev = pt->dev; 179 struct dma_pool *dma_pool; 180 int ret; 181 182 /* Allocate a dma pool for the queue */ 183 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev)); 184 185 dma_pool = dma_pool_create(dma_pool_name, dev, 186 PT_DMAPOOL_MAX_SIZE, 187 PT_DMAPOOL_ALIGN, 0); 188 if (!dma_pool) 189 return -ENOMEM; 190 191 /* ptdma core initialisation */ 192 iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET); 193 iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET); 194 iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET); 195 iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET); 196 iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET); 197 198 cmd_q->pt = pt; 199 cmd_q->dma_pool = dma_pool; 200 mutex_init(&cmd_q->q_mutex); 201 202 /* Page alignment satisfies our needs for N <= 128 */ 203 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); 204 cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize, 205 &cmd_q->qbase_dma, 206 GFP_KERNEL); 207 if (!cmd_q->qbase) { 208 dev_err(dev, "unable to allocate command queue\n"); 209 ret = -ENOMEM; 210 goto e_dma_alloc; 211 } 212 213 cmd_q->qidx = 0; 214 215 /* Preset some register values */ 216 cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR; 217 218 /* Turn off the queues and disable interrupts until ready */ 219 pt_core_disable_queue_interrupts(pt); 220 221 cmd_q->qcontrol = 0; /* Start with nothing */ 222 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 223 224 ioread32(cmd_q->reg_control + 0x0104); 225 ioread32(cmd_q->reg_control + 0x0100); 226 227 /* Clear the interrupt status */ 228 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); 229 230 /* Request an irq */ 231 ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt); 232 if (ret) 233 goto e_pool; 234 235 /* Update the device registers with queue information. */ 236 cmd_q->qcontrol &= ~CMD_Q_SIZE; 237 cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL); 238 239 cmd_q->qdma_tail = cmd_q->qbase_dma; 240 dma_addr_lo = lower_32_bits(cmd_q->qdma_tail); 241 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004); 242 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008); 243 244 dma_addr_hi = upper_32_bits(cmd_q->qdma_tail); 245 cmd_q->qcontrol |= (dma_addr_hi << 16); 246 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 247 248 pt_core_enable_queue_interrupts(pt); 249 250 /* Register the DMA engine support */ 251 ret = pt_dmaengine_register(pt); 252 if (ret) 253 goto e_dmaengine; 254 255 /* Set up debugfs entries */ 256 ptdma_debugfs_setup(pt); 257 258 return 0; 259 260 e_dmaengine: 261 free_irq(pt->pt_irq, pt); 262 263 e_dma_alloc: 264 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma); 265 266 e_pool: 267 dev_err(dev, "unable to allocate an IRQ\n"); 268 dma_pool_destroy(pt->cmd_q.dma_pool); 269 270 return ret; 271 } 272 273 void pt_core_destroy(struct pt_device *pt) 274 { 275 struct device *dev = pt->dev; 276 struct pt_cmd_queue *cmd_q = &pt->cmd_q; 277 struct pt_cmd *cmd; 278 279 /* Unregister the DMA engine */ 280 pt_dmaengine_unregister(pt); 281 282 /* Disable and clear interrupts */ 283 pt_core_disable_queue_interrupts(pt); 284 285 /* Turn off the run bit */ 286 pt_stop_queue(cmd_q); 287 288 /* Clear the interrupt status */ 289 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); 290 ioread32(cmd_q->reg_control + 0x0104); 291 ioread32(cmd_q->reg_control + 0x0100); 292 293 free_irq(pt->pt_irq, pt); 294 295 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, 296 cmd_q->qbase_dma); 297 298 /* Flush the cmd queue */ 299 while (!list_empty(&pt->cmd)) { 300 /* Invoke the callback directly with an error code */ 301 cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry); 302 list_del(&cmd->entry); 303 cmd->pt_cmd_callback(cmd->data, -ENODEV); 304 } 305 } 306