1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x Command DMA 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 */ 7 8 #include <linux/slab.h> 9 #include <linux/scatterlist.h> 10 #include <linux/dma-mapping.h> 11 12 #include "../cdma.h" 13 #include "../channel.h" 14 #include "../dev.h" 15 #include "../debug.h" 16 17 /* 18 * Put the restart at the end of pushbuffer memory 19 */ 20 static void push_buffer_init(struct push_buffer *pb) 21 { 22 *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0); 23 } 24 25 /* 26 * Increment timedout buffer's syncpt via CPU. 27 */ 28 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, 29 u32 syncpt_incrs, u32 syncval, u32 nr_slots) 30 { 31 unsigned int i; 32 33 for (i = 0; i < syncpt_incrs; i++) 34 host1x_syncpt_incr(cdma->timeout.syncpt); 35 36 /* after CPU incr, ensure shadow is up to date */ 37 host1x_syncpt_load(cdma->timeout.syncpt); 38 } 39 40 /* 41 * Start channel DMA 42 */ 43 static void cdma_start(struct host1x_cdma *cdma) 44 { 45 struct host1x_channel *ch = cdma_to_channel(cdma); 46 u64 start, end; 47 48 if (cdma->running) 49 return; 50 51 cdma->last_pos = cdma->push_buffer.pos; 52 start = cdma->push_buffer.dma; 53 end = cdma->push_buffer.size + 4; 54 55 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 56 HOST1X_CHANNEL_DMACTRL); 57 58 /* set base, put and end pointer */ 59 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART); 60 #if HOST1X_HW >= 6 61 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI); 62 #endif 63 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 64 #if HOST1X_HW >= 6 65 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI); 66 #endif 67 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND); 68 #if HOST1X_HW >= 6 69 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI); 70 #endif 71 72 /* reset GET */ 73 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 74 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 75 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 76 HOST1X_CHANNEL_DMACTRL); 77 78 /* start the command DMA */ 79 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 80 81 cdma->running = true; 82 } 83 84 /* 85 * Similar to cdma_start(), but rather than starting from an idle 86 * state (where DMA GET is set to DMA PUT), on a timeout we restore 87 * DMA GET from an explicit value (so DMA may again be pending). 88 */ 89 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) 90 { 91 struct host1x *host1x = cdma_to_host1x(cdma); 92 struct host1x_channel *ch = cdma_to_channel(cdma); 93 u64 start, end; 94 95 if (cdma->running) 96 return; 97 98 cdma->last_pos = cdma->push_buffer.pos; 99 100 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 101 HOST1X_CHANNEL_DMACTRL); 102 103 start = cdma->push_buffer.dma; 104 end = cdma->push_buffer.size + 4; 105 106 /* set base, end pointer (all of memory) */ 107 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART); 108 #if HOST1X_HW >= 6 109 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI); 110 #endif 111 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND); 112 #if HOST1X_HW >= 6 113 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI); 114 #endif 115 116 /* set GET, by loading the value in PUT (then reset GET) */ 117 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT); 118 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 119 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 120 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 121 HOST1X_CHANNEL_DMACTRL); 122 123 dev_dbg(host1x->dev, 124 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__, 125 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 126 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 127 cdma->last_pos); 128 129 /* deassert GET reset and set PUT */ 130 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 131 HOST1X_CHANNEL_DMACTRL); 132 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 133 134 /* start the command DMA */ 135 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 136 137 cdma->running = true; 138 } 139 140 /* 141 * Kick channel DMA into action by writing its PUT offset (if it has changed) 142 */ 143 static void cdma_flush(struct host1x_cdma *cdma) 144 { 145 struct host1x_channel *ch = cdma_to_channel(cdma); 146 147 if (cdma->push_buffer.pos != cdma->last_pos) { 148 host1x_ch_writel(ch, cdma->push_buffer.pos, 149 HOST1X_CHANNEL_DMAPUT); 150 cdma->last_pos = cdma->push_buffer.pos; 151 } 152 } 153 154 static void cdma_stop(struct host1x_cdma *cdma) 155 { 156 struct host1x_channel *ch = cdma_to_channel(cdma); 157 158 mutex_lock(&cdma->lock); 159 160 if (cdma->running) { 161 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY); 162 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 163 HOST1X_CHANNEL_DMACTRL); 164 cdma->running = false; 165 } 166 167 mutex_unlock(&cdma->lock); 168 } 169 170 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch, 171 bool stop) 172 { 173 #if HOST1X_HW >= 6 174 host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP); 175 #else 176 u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP); 177 if (stop) 178 cmdproc_stop |= BIT(ch->id); 179 else 180 cmdproc_stop &= ~BIT(ch->id); 181 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 182 #endif 183 } 184 185 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch) 186 { 187 #if HOST1X_HW >= 6 188 host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN); 189 #else 190 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN); 191 #endif 192 } 193 194 /* 195 * Stops both channel's command processor and CDMA immediately. 196 * Also, tears down the channel and resets corresponding module. 197 */ 198 static void cdma_freeze(struct host1x_cdma *cdma) 199 { 200 struct host1x *host = cdma_to_host1x(cdma); 201 struct host1x_channel *ch = cdma_to_channel(cdma); 202 203 if (cdma->torndown && !cdma->running) { 204 dev_warn(host->dev, "Already torn down\n"); 205 return; 206 } 207 208 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id); 209 210 cdma_hw_cmdproc_stop(host, ch, true); 211 212 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", 213 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 214 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 215 cdma->last_pos); 216 217 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 218 HOST1X_CHANNEL_DMACTRL); 219 220 cdma_hw_teardown(host, ch); 221 222 cdma->running = false; 223 cdma->torndown = true; 224 } 225 226 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) 227 { 228 struct host1x *host1x = cdma_to_host1x(cdma); 229 struct host1x_channel *ch = cdma_to_channel(cdma); 230 231 dev_dbg(host1x->dev, 232 "resuming channel (id %u, DMAGET restart = 0x%x)\n", 233 ch->id, getptr); 234 235 cdma_hw_cmdproc_stop(host1x, ch, false); 236 237 cdma->torndown = false; 238 cdma_timeout_restart(cdma, getptr); 239 } 240 241 /* 242 * If this timeout fires, it indicates the current sync_queue entry has 243 * exceeded its TTL and the userctx should be timed out and remaining 244 * submits already issued cleaned up (future submits return an error). 245 */ 246 static void cdma_timeout_handler(struct work_struct *work) 247 { 248 u32 syncpt_val; 249 struct host1x_cdma *cdma; 250 struct host1x *host1x; 251 struct host1x_channel *ch; 252 253 cdma = container_of(to_delayed_work(work), struct host1x_cdma, 254 timeout.wq); 255 host1x = cdma_to_host1x(cdma); 256 ch = cdma_to_channel(cdma); 257 258 host1x_debug_dump(cdma_to_host1x(cdma)); 259 260 mutex_lock(&cdma->lock); 261 262 if (!cdma->timeout.client) { 263 dev_dbg(host1x->dev, 264 "cdma_timeout: expired, but has no clientid\n"); 265 mutex_unlock(&cdma->lock); 266 return; 267 } 268 269 /* stop processing to get a clean snapshot */ 270 cdma_hw_cmdproc_stop(host1x, ch, true); 271 272 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 273 274 /* has buffer actually completed? */ 275 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { 276 dev_dbg(host1x->dev, 277 "cdma_timeout: expired, but buffer had completed\n"); 278 /* restore */ 279 cdma_hw_cmdproc_stop(host1x, ch, false); 280 mutex_unlock(&cdma->lock); 281 return; 282 } 283 284 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n", 285 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, 286 syncpt_val, cdma->timeout.syncpt_val); 287 288 /* stop HW, resetting channel/module */ 289 host1x_hw_cdma_freeze(host1x, cdma); 290 291 host1x_cdma_update_sync_queue(cdma, ch->dev); 292 mutex_unlock(&cdma->lock); 293 } 294 295 /* 296 * Init timeout resources 297 */ 298 static int cdma_timeout_init(struct host1x_cdma *cdma) 299 { 300 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); 301 cdma->timeout.initialized = true; 302 303 return 0; 304 } 305 306 /* 307 * Clean up timeout resources 308 */ 309 static void cdma_timeout_destroy(struct host1x_cdma *cdma) 310 { 311 if (cdma->timeout.initialized) 312 cancel_delayed_work(&cdma->timeout.wq); 313 314 cdma->timeout.initialized = false; 315 } 316 317 static const struct host1x_cdma_ops host1x_cdma_ops = { 318 .start = cdma_start, 319 .stop = cdma_stop, 320 .flush = cdma_flush, 321 322 .timeout_init = cdma_timeout_init, 323 .timeout_destroy = cdma_timeout_destroy, 324 .freeze = cdma_freeze, 325 .resume = cdma_resume, 326 .timeout_cpu_incr = cdma_timeout_cpu_incr, 327 }; 328 329 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { 330 .init = push_buffer_init, 331 }; 332