1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x Command DMA 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 */ 7 8 #include <linux/slab.h> 9 #include <linux/scatterlist.h> 10 #include <linux/dma-mapping.h> 11 12 #include "../cdma.h" 13 #include "../channel.h" 14 #include "../dev.h" 15 #include "../debug.h" 16 17 /* 18 * Put the restart at the end of pushbuffer memory 19 */ 20 static void push_buffer_init(struct push_buffer *pb) 21 { 22 *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0); 23 } 24 25 /* 26 * Increment timedout buffer's syncpt via CPU. 27 */ 28 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, 29 u32 syncpt_incrs, u32 syncval, u32 nr_slots) 30 { 31 unsigned int i; 32 33 for (i = 0; i < syncpt_incrs; i++) 34 host1x_syncpt_incr(cdma->timeout.syncpt); 35 36 /* after CPU incr, ensure shadow is up to date */ 37 host1x_syncpt_load(cdma->timeout.syncpt); 38 } 39 40 /* 41 * Start channel DMA 42 */ 43 static void cdma_start(struct host1x_cdma *cdma) 44 { 45 struct host1x_channel *ch = cdma_to_channel(cdma); 46 u64 start, end; 47 48 if (cdma->running) 49 return; 50 51 cdma->last_pos = cdma->push_buffer.pos; 52 start = cdma->push_buffer.dma; 53 end = cdma->push_buffer.size + 4; 54 55 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 56 HOST1X_CHANNEL_DMACTRL); 57 58 /* set base, put and end pointer */ 59 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART); 60 #if HOST1X_HW >= 6 61 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI); 62 #endif 63 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 64 #if HOST1X_HW >= 6 65 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI); 66 #endif 67 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND); 68 #if HOST1X_HW >= 6 69 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI); 70 #endif 71 72 /* reset GET */ 73 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 74 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 75 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 76 HOST1X_CHANNEL_DMACTRL); 77 78 /* start the command DMA */ 79 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 80 81 cdma->running = true; 82 } 83 84 /* 85 * Similar to cdma_start(), but rather than starting from an idle 86 * state (where DMA GET is set to DMA PUT), on a timeout we restore 87 * DMA GET from an explicit value (so DMA may again be pending). 88 */ 89 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) 90 { 91 struct host1x *host1x = cdma_to_host1x(cdma); 92 struct host1x_channel *ch = cdma_to_channel(cdma); 93 u64 start, end; 94 95 if (cdma->running) 96 return; 97 98 cdma->last_pos = cdma->push_buffer.pos; 99 100 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 101 HOST1X_CHANNEL_DMACTRL); 102 103 start = cdma->push_buffer.dma; 104 end = cdma->push_buffer.size + 4; 105 106 /* set base, end pointer (all of memory) */ 107 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART); 108 #if HOST1X_HW >= 6 109 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI); 110 #endif 111 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND); 112 #if HOST1X_HW >= 6 113 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI); 114 #endif 115 116 /* set GET, by loading the value in PUT (then reset GET) */ 117 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT); 118 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 119 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 120 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 121 HOST1X_CHANNEL_DMACTRL); 122 123 dev_dbg(host1x->dev, 124 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__, 125 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 126 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 127 cdma->last_pos); 128 129 /* deassert GET reset and set PUT */ 130 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 131 HOST1X_CHANNEL_DMACTRL); 132 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 133 134 /* start the command DMA */ 135 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 136 137 cdma->running = true; 138 } 139 140 /* 141 * Kick channel DMA into action by writing its PUT offset (if it has changed) 142 */ 143 static void cdma_flush(struct host1x_cdma *cdma) 144 { 145 struct host1x_channel *ch = cdma_to_channel(cdma); 146 147 if (cdma->push_buffer.pos != cdma->last_pos) { 148 host1x_ch_writel(ch, cdma->push_buffer.pos, 149 HOST1X_CHANNEL_DMAPUT); 150 cdma->last_pos = cdma->push_buffer.pos; 151 } 152 } 153 154 static void cdma_stop(struct host1x_cdma *cdma) 155 { 156 struct host1x_channel *ch = cdma_to_channel(cdma); 157 158 mutex_lock(&cdma->lock); 159 160 if (cdma->running) { 161 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY); 162 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 163 HOST1X_CHANNEL_DMACTRL); 164 cdma->running = false; 165 } 166 167 mutex_unlock(&cdma->lock); 168 } 169 170 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch, 171 bool stop) 172 { 173 #if HOST1X_HW >= 6 174 host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP); 175 #else 176 u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP); 177 if (stop) 178 cmdproc_stop |= BIT(ch->id); 179 else 180 cmdproc_stop &= ~BIT(ch->id); 181 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 182 #endif 183 } 184 185 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch) 186 { 187 #if HOST1X_HW >= 6 188 host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN); 189 #else 190 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN); 191 #endif 192 } 193 194 /* 195 * Stops both channel's command processor and CDMA immediately. 196 * Also, tears down the channel and resets corresponding module. 197 */ 198 static void cdma_freeze(struct host1x_cdma *cdma) 199 { 200 struct host1x *host = cdma_to_host1x(cdma); 201 struct host1x_channel *ch = cdma_to_channel(cdma); 202 203 if (cdma->torndown && !cdma->running) { 204 dev_warn(host->dev, "Already torn down\n"); 205 return; 206 } 207 208 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id); 209 210 cdma_hw_cmdproc_stop(host, ch, true); 211 212 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", 213 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 214 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 215 cdma->last_pos); 216 217 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 218 HOST1X_CHANNEL_DMACTRL); 219 220 cdma_hw_teardown(host, ch); 221 222 cdma->running = false; 223 cdma->torndown = true; 224 } 225 226 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) 227 { 228 struct host1x *host1x = cdma_to_host1x(cdma); 229 struct host1x_channel *ch = cdma_to_channel(cdma); 230 231 dev_dbg(host1x->dev, 232 "resuming channel (id %u, DMAGET restart = 0x%x)\n", 233 ch->id, getptr); 234 235 cdma_hw_cmdproc_stop(host1x, ch, false); 236 237 cdma->torndown = false; 238 cdma_timeout_restart(cdma, getptr); 239 } 240 241 static void timeout_release_mlock(struct host1x_cdma *cdma) 242 { 243 #if HOST1X_HW >= 8 244 /* Tegra186 and Tegra194 require a more complicated MLOCK release 245 * sequence. Furthermore, those chips by default don't enforce MLOCKs, 246 * so it turns out that if we don't /actually/ need MLOCKs, we can just 247 * ignore them. 248 * 249 * As such, for now just implement this on Tegra234 where things are 250 * stricter but also easy to implement. 251 */ 252 struct host1x_channel *ch = cdma_to_channel(cdma); 253 struct host1x *host1x = cdma_to_host1x(cdma); 254 u32 offset; 255 256 switch (ch->client->class) { 257 case HOST1X_CLASS_VIC: 258 offset = HOST1X_COMMON_VIC_MLOCK; 259 break; 260 case HOST1X_CLASS_NVDEC: 261 offset = HOST1X_COMMON_NVDEC_MLOCK; 262 break; 263 default: 264 WARN(1, "%s was not updated for class %u", __func__, ch->client->class); 265 return; 266 } 267 268 host1x_common_writel(host1x, 0x0, offset); 269 #endif 270 } 271 272 /* 273 * If this timeout fires, it indicates the current sync_queue entry has 274 * exceeded its TTL and the userctx should be timed out and remaining 275 * submits already issued cleaned up (future submits return an error). 276 */ 277 static void cdma_timeout_handler(struct work_struct *work) 278 { 279 u32 syncpt_val; 280 struct host1x_cdma *cdma; 281 struct host1x *host1x; 282 struct host1x_channel *ch; 283 284 cdma = container_of(to_delayed_work(work), struct host1x_cdma, 285 timeout.wq); 286 host1x = cdma_to_host1x(cdma); 287 ch = cdma_to_channel(cdma); 288 289 host1x_debug_dump(cdma_to_host1x(cdma)); 290 291 mutex_lock(&cdma->lock); 292 293 if (!cdma->timeout.client) { 294 dev_dbg(host1x->dev, 295 "cdma_timeout: expired, but has no clientid\n"); 296 mutex_unlock(&cdma->lock); 297 return; 298 } 299 300 /* stop processing to get a clean snapshot */ 301 cdma_hw_cmdproc_stop(host1x, ch, true); 302 303 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 304 305 /* has buffer actually completed? */ 306 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { 307 dev_dbg(host1x->dev, 308 "cdma_timeout: expired, but buffer had completed\n"); 309 /* restore */ 310 cdma_hw_cmdproc_stop(host1x, ch, false); 311 mutex_unlock(&cdma->lock); 312 return; 313 } 314 315 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n", 316 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, 317 syncpt_val, cdma->timeout.syncpt_val); 318 319 /* stop HW, resetting channel/module */ 320 host1x_hw_cdma_freeze(host1x, cdma); 321 322 /* release any held MLOCK */ 323 timeout_release_mlock(cdma); 324 325 host1x_cdma_update_sync_queue(cdma, ch->dev); 326 mutex_unlock(&cdma->lock); 327 } 328 329 /* 330 * Init timeout resources 331 */ 332 static int cdma_timeout_init(struct host1x_cdma *cdma) 333 { 334 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); 335 cdma->timeout.initialized = true; 336 337 return 0; 338 } 339 340 /* 341 * Clean up timeout resources 342 */ 343 static void cdma_timeout_destroy(struct host1x_cdma *cdma) 344 { 345 if (cdma->timeout.initialized) 346 cancel_delayed_work(&cdma->timeout.wq); 347 348 cdma->timeout.initialized = false; 349 } 350 351 static const struct host1x_cdma_ops host1x_cdma_ops = { 352 .start = cdma_start, 353 .stop = cdma_stop, 354 .flush = cdma_flush, 355 356 .timeout_init = cdma_timeout_init, 357 .timeout_destroy = cdma_timeout_destroy, 358 .freeze = cdma_freeze, 359 .resume = cdma_resume, 360 .timeout_cpu_incr = cdma_timeout_cpu_incr, 361 }; 362 363 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { 364 .init = push_buffer_init, 365 }; 366