1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Passthru DMA device driver 4 * -- Based on the CCP driver 5 * 6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. 7 * 8 * Author: Sanjay R Mehta <sanju.mehta@amd.com> 9 * Author: Gary R Hook <gary.hook@amd.com> 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/debugfs.h> 15 #include <linux/interrupt.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/pci.h> 19 20 #include "ptdma.h" 21 22 /* Human-readable error strings */ 23 static char *pt_error_codes[] = { 24 "", 25 "ERR 01: ILLEGAL_ENGINE", 26 "ERR 03: ILLEGAL_FUNCTION_TYPE", 27 "ERR 04: ILLEGAL_FUNCTION_MODE", 28 "ERR 06: ILLEGAL_FUNCTION_SIZE", 29 "ERR 08: ILLEGAL_FUNCTION_RSVD", 30 "ERR 09: ILLEGAL_BUFFER_LENGTH", 31 "ERR 10: VLSB_FAULT", 32 "ERR 11: ILLEGAL_MEM_ADDR", 33 "ERR 12: ILLEGAL_MEM_SEL", 34 "ERR 13: ILLEGAL_CONTEXT_ID", 35 "ERR 15: 0xF Reserved", 36 "ERR 18: CMD_TIMEOUT", 37 "ERR 19: IDMA0_AXI_SLVERR", 38 "ERR 20: IDMA0_AXI_DECERR", 39 "ERR 21: 0x15 Reserved", 40 "ERR 22: IDMA1_AXI_SLAVE_FAULT", 41 "ERR 23: IDMA1_AIXI_DECERR", 42 "ERR 24: 0x18 Reserved", 43 "ERR 27: 0x1B Reserved", 44 "ERR 38: ODMA0_AXI_SLVERR", 45 "ERR 39: ODMA0_AXI_DECERR", 46 "ERR 40: 0x28 Reserved", 47 "ERR 41: ODMA1_AXI_SLVERR", 48 "ERR 42: ODMA1_AXI_DECERR", 49 "ERR 43: LSB_PARITY_ERR", 50 }; 51 52 static void pt_log_error(struct pt_device *d, int e) 53 { 54 dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e); 55 } 56 57 void pt_start_queue(struct pt_cmd_queue *cmd_q) 58 { 59 /* Turn on the run bit */ 60 iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control); 61 } 62 63 void pt_stop_queue(struct pt_cmd_queue *cmd_q) 64 { 65 /* Turn off the run bit */ 66 iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control); 67 } 68 69 static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q) 70 { 71 bool soc = FIELD_GET(DWORD0_SOC, desc->dw0); 72 u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx]; 73 u32 tail; 74 75 if (soc) { 76 desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0); 77 desc->dw0 &= ~DWORD0_SOC; 78 } 79 mutex_lock(&cmd_q->q_mutex); 80 81 /* Copy 32-byte command descriptor to hw queue. */ 82 memcpy(q_desc, desc, 32); 83 cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN; 84 85 /* The data used by this command must be flushed to memory */ 86 wmb(); 87 88 /* Write the new tail address back to the queue register */ 89 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); 90 iowrite32(tail, cmd_q->reg_control + 0x0004); 91 92 /* Turn the queue back on using our cached control register */ 93 pt_start_queue(cmd_q); 94 mutex_unlock(&cmd_q->q_mutex); 95 96 return 0; 97 } 98 99 int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, 100 struct pt_passthru_engine *pt_engine) 101 { 102 struct ptdma_desc desc; 103 104 cmd_q->cmd_error = 0; 105 memset(&desc, 0, sizeof(desc)); 106 desc.dw0 = CMD_DESC_DW0_VAL; 107 desc.length = pt_engine->src_len; 108 desc.src_lo = lower_32_bits(pt_engine->src_dma); 109 desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma); 110 desc.dst_lo = lower_32_bits(pt_engine->dst_dma); 111 desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma); 112 113 return pt_core_execute_cmd(&desc, cmd_q); 114 } 115 116 static inline void pt_core_disable_queue_interrupts(struct pt_device *pt) 117 { 118 iowrite32(0, pt->cmd_q.reg_control + 0x000C); 119 } 120 121 static inline void pt_core_enable_queue_interrupts(struct pt_device *pt) 122 { 123 iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C); 124 } 125 126 static void pt_do_cmd_complete(unsigned long data) 127 { 128 struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data; 129 struct pt_cmd *cmd = tdata->cmd; 130 struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q; 131 u32 tail; 132 133 if (cmd_q->cmd_error) { 134 /* 135 * Log the error and flush the queue by 136 * moving the head pointer 137 */ 138 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); 139 pt_log_error(cmd_q->pt, cmd_q->cmd_error); 140 iowrite32(tail, cmd_q->reg_control + 0x0008); 141 } 142 143 cmd->pt_cmd_callback(cmd->data, cmd->ret); 144 } 145 146 static irqreturn_t pt_core_irq_handler(int irq, void *data) 147 { 148 struct pt_device *pt = data; 149 struct pt_cmd_queue *cmd_q = &pt->cmd_q; 150 u32 status; 151 152 pt_core_disable_queue_interrupts(pt); 153 status = ioread32(cmd_q->reg_control + 0x0010); 154 if (status) { 155 cmd_q->int_status = status; 156 cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100); 157 cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104); 158 159 /* On error, only save the first error value */ 160 if ((status & INT_ERROR) && !cmd_q->cmd_error) 161 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); 162 163 /* Acknowledge the interrupt */ 164 iowrite32(status, cmd_q->reg_control + 0x0010); 165 pt_core_enable_queue_interrupts(pt); 166 pt_do_cmd_complete((ulong)&pt->tdata); 167 } 168 return IRQ_HANDLED; 169 } 170 171 int pt_core_init(struct pt_device *pt) 172 { 173 char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; 174 struct pt_cmd_queue *cmd_q = &pt->cmd_q; 175 u32 dma_addr_lo, dma_addr_hi; 176 struct device *dev = pt->dev; 177 struct dma_pool *dma_pool; 178 int ret; 179 180 /* Allocate a dma pool for the queue */ 181 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev)); 182 183 dma_pool = dma_pool_create(dma_pool_name, dev, 184 PT_DMAPOOL_MAX_SIZE, 185 PT_DMAPOOL_ALIGN, 0); 186 if (!dma_pool) 187 return -ENOMEM; 188 189 /* ptdma core initialisation */ 190 iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET); 191 iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET); 192 iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET); 193 iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET); 194 iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET); 195 196 cmd_q->pt = pt; 197 cmd_q->dma_pool = dma_pool; 198 mutex_init(&cmd_q->q_mutex); 199 200 /* Page alignment satisfies our needs for N <= 128 */ 201 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); 202 cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize, 203 &cmd_q->qbase_dma, 204 GFP_KERNEL); 205 if (!cmd_q->qbase) { 206 dev_err(dev, "unable to allocate command queue\n"); 207 ret = -ENOMEM; 208 goto e_dma_alloc; 209 } 210 211 cmd_q->qidx = 0; 212 213 /* Preset some register values */ 214 cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR; 215 216 /* Turn off the queues and disable interrupts until ready */ 217 pt_core_disable_queue_interrupts(pt); 218 219 cmd_q->qcontrol = 0; /* Start with nothing */ 220 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 221 222 ioread32(cmd_q->reg_control + 0x0104); 223 ioread32(cmd_q->reg_control + 0x0100); 224 225 /* Clear the interrupt status */ 226 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); 227 228 /* Request an irq */ 229 ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt); 230 if (ret) 231 goto e_pool; 232 233 /* Update the device registers with queue information. */ 234 cmd_q->qcontrol &= ~CMD_Q_SIZE; 235 cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL); 236 237 cmd_q->qdma_tail = cmd_q->qbase_dma; 238 dma_addr_lo = lower_32_bits(cmd_q->qdma_tail); 239 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004); 240 iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008); 241 242 dma_addr_hi = upper_32_bits(cmd_q->qdma_tail); 243 cmd_q->qcontrol |= (dma_addr_hi << 16); 244 iowrite32(cmd_q->qcontrol, cmd_q->reg_control); 245 246 pt_core_enable_queue_interrupts(pt); 247 248 /* Register the DMA engine support */ 249 ret = pt_dmaengine_register(pt); 250 if (ret) 251 goto e_dmaengine; 252 253 return 0; 254 255 e_dmaengine: 256 free_irq(pt->pt_irq, pt); 257 258 e_dma_alloc: 259 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma); 260 261 e_pool: 262 dev_err(dev, "unable to allocate an IRQ\n"); 263 dma_pool_destroy(pt->cmd_q.dma_pool); 264 265 return ret; 266 } 267 268 void pt_core_destroy(struct pt_device *pt) 269 { 270 struct device *dev = pt->dev; 271 struct pt_cmd_queue *cmd_q = &pt->cmd_q; 272 struct pt_cmd *cmd; 273 274 /* Unregister the DMA engine */ 275 pt_dmaengine_unregister(pt); 276 277 /* Disable and clear interrupts */ 278 pt_core_disable_queue_interrupts(pt); 279 280 /* Turn off the run bit */ 281 pt_stop_queue(cmd_q); 282 283 /* Clear the interrupt status */ 284 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); 285 ioread32(cmd_q->reg_control + 0x0104); 286 ioread32(cmd_q->reg_control + 0x0100); 287 288 free_irq(pt->pt_irq, pt); 289 290 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, 291 cmd_q->qbase_dma); 292 293 /* Flush the cmd queue */ 294 while (!list_empty(&pt->cmd)) { 295 /* Invoke the callback directly with an error code */ 296 cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry); 297 list_del(&cmd->entry); 298 cmd->pt_cmd_callback(cmd->data, -ENODEV); 299 } 300 } 301