1 /* 2 * Tegra host1x Command DMA 3 * 4 * Copyright (c) 2010-2013, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/slab.h> 20 #include <linux/scatterlist.h> 21 #include <linux/dma-mapping.h> 22 23 #include "../cdma.h" 24 #include "../channel.h" 25 #include "../dev.h" 26 #include "../debug.h" 27 28 /* 29 * Put the restart at the end of pushbuffer memory 30 */ 31 static void push_buffer_init(struct push_buffer *pb) 32 { 33 *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0); 34 } 35 36 /* 37 * Increment timedout buffer's syncpt via CPU. 38 */ 39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, 40 u32 syncpt_incrs, u32 syncval, u32 nr_slots) 41 { 42 unsigned int i; 43 44 for (i = 0; i < syncpt_incrs; i++) 45 host1x_syncpt_incr(cdma->timeout.syncpt); 46 47 /* after CPU incr, ensure shadow is up to date */ 48 host1x_syncpt_load(cdma->timeout.syncpt); 49 } 50 51 /* 52 * Start channel DMA 53 */ 54 static void cdma_start(struct host1x_cdma *cdma) 55 { 56 struct host1x_channel *ch = cdma_to_channel(cdma); 57 u64 start, end; 58 59 if (cdma->running) 60 return; 61 62 cdma->last_pos = cdma->push_buffer.pos; 63 start = cdma->push_buffer.dma; 64 end = cdma->push_buffer.size + 4; 65 66 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 67 HOST1X_CHANNEL_DMACTRL); 68 69 /* set base, put and end pointer */ 70 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART); 71 #if HOST1X_HW >= 6 72 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI); 73 #endif 74 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 75 #if HOST1X_HW >= 6 76 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI); 77 #endif 78 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND); 79 #if HOST1X_HW >= 6 80 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI); 81 #endif 82 83 /* reset GET */ 84 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 85 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 86 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 87 HOST1X_CHANNEL_DMACTRL); 88 89 /* start the command DMA */ 90 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 91 92 cdma->running = true; 93 } 94 95 /* 96 * Similar to cdma_start(), but rather than starting from an idle 97 * state (where DMA GET is set to DMA PUT), on a timeout we restore 98 * DMA GET from an explicit value (so DMA may again be pending). 99 */ 100 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) 101 { 102 struct host1x *host1x = cdma_to_host1x(cdma); 103 struct host1x_channel *ch = cdma_to_channel(cdma); 104 u64 start, end; 105 106 if (cdma->running) 107 return; 108 109 cdma->last_pos = cdma->push_buffer.pos; 110 111 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 112 HOST1X_CHANNEL_DMACTRL); 113 114 start = cdma->push_buffer.dma; 115 end = cdma->push_buffer.size + 4; 116 117 /* set base, end pointer (all of memory) */ 118 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART); 119 #if HOST1X_HW >= 6 120 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI); 121 #endif 122 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND); 123 #if HOST1X_HW >= 6 124 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI); 125 #endif 126 127 /* set GET, by loading the value in PUT (then reset GET) */ 128 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT); 129 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 130 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 131 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 132 HOST1X_CHANNEL_DMACTRL); 133 134 dev_dbg(host1x->dev, 135 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__, 136 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 137 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 138 cdma->last_pos); 139 140 /* deassert GET reset and set PUT */ 141 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 142 HOST1X_CHANNEL_DMACTRL); 143 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 144 145 /* start the command DMA */ 146 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 147 148 cdma->running = true; 149 } 150 151 /* 152 * Kick channel DMA into action by writing its PUT offset (if it has changed) 153 */ 154 static void cdma_flush(struct host1x_cdma *cdma) 155 { 156 struct host1x_channel *ch = cdma_to_channel(cdma); 157 158 if (cdma->push_buffer.pos != cdma->last_pos) { 159 host1x_ch_writel(ch, cdma->push_buffer.pos, 160 HOST1X_CHANNEL_DMAPUT); 161 cdma->last_pos = cdma->push_buffer.pos; 162 } 163 } 164 165 static void cdma_stop(struct host1x_cdma *cdma) 166 { 167 struct host1x_channel *ch = cdma_to_channel(cdma); 168 169 mutex_lock(&cdma->lock); 170 171 if (cdma->running) { 172 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY); 173 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 174 HOST1X_CHANNEL_DMACTRL); 175 cdma->running = false; 176 } 177 178 mutex_unlock(&cdma->lock); 179 } 180 181 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch, 182 bool stop) 183 { 184 #if HOST1X_HW >= 6 185 host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP); 186 #else 187 u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP); 188 if (stop) 189 cmdproc_stop |= BIT(ch->id); 190 else 191 cmdproc_stop &= ~BIT(ch->id); 192 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 193 #endif 194 } 195 196 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch) 197 { 198 #if HOST1X_HW >= 6 199 host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN); 200 #else 201 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN); 202 #endif 203 } 204 205 /* 206 * Stops both channel's command processor and CDMA immediately. 207 * Also, tears down the channel and resets corresponding module. 208 */ 209 static void cdma_freeze(struct host1x_cdma *cdma) 210 { 211 struct host1x *host = cdma_to_host1x(cdma); 212 struct host1x_channel *ch = cdma_to_channel(cdma); 213 214 if (cdma->torndown && !cdma->running) { 215 dev_warn(host->dev, "Already torn down\n"); 216 return; 217 } 218 219 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id); 220 221 cdma_hw_cmdproc_stop(host, ch, true); 222 223 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", 224 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 225 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 226 cdma->last_pos); 227 228 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 229 HOST1X_CHANNEL_DMACTRL); 230 231 cdma_hw_teardown(host, ch); 232 233 cdma->running = false; 234 cdma->torndown = true; 235 } 236 237 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) 238 { 239 struct host1x *host1x = cdma_to_host1x(cdma); 240 struct host1x_channel *ch = cdma_to_channel(cdma); 241 242 dev_dbg(host1x->dev, 243 "resuming channel (id %u, DMAGET restart = 0x%x)\n", 244 ch->id, getptr); 245 246 cdma_hw_cmdproc_stop(host1x, ch, false); 247 248 cdma->torndown = false; 249 cdma_timeout_restart(cdma, getptr); 250 } 251 252 /* 253 * If this timeout fires, it indicates the current sync_queue entry has 254 * exceeded its TTL and the userctx should be timed out and remaining 255 * submits already issued cleaned up (future submits return an error). 256 */ 257 static void cdma_timeout_handler(struct work_struct *work) 258 { 259 u32 syncpt_val; 260 struct host1x_cdma *cdma; 261 struct host1x *host1x; 262 struct host1x_channel *ch; 263 264 cdma = container_of(to_delayed_work(work), struct host1x_cdma, 265 timeout.wq); 266 host1x = cdma_to_host1x(cdma); 267 ch = cdma_to_channel(cdma); 268 269 host1x_debug_dump(cdma_to_host1x(cdma)); 270 271 mutex_lock(&cdma->lock); 272 273 if (!cdma->timeout.client) { 274 dev_dbg(host1x->dev, 275 "cdma_timeout: expired, but has no clientid\n"); 276 mutex_unlock(&cdma->lock); 277 return; 278 } 279 280 /* stop processing to get a clean snapshot */ 281 cdma_hw_cmdproc_stop(host1x, ch, true); 282 283 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 284 285 /* has buffer actually completed? */ 286 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { 287 dev_dbg(host1x->dev, 288 "cdma_timeout: expired, but buffer had completed\n"); 289 /* restore */ 290 cdma_hw_cmdproc_stop(host1x, ch, false); 291 mutex_unlock(&cdma->lock); 292 return; 293 } 294 295 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n", 296 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, 297 syncpt_val, cdma->timeout.syncpt_val); 298 299 /* stop HW, resetting channel/module */ 300 host1x_hw_cdma_freeze(host1x, cdma); 301 302 host1x_cdma_update_sync_queue(cdma, ch->dev); 303 mutex_unlock(&cdma->lock); 304 } 305 306 /* 307 * Init timeout resources 308 */ 309 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt) 310 { 311 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); 312 cdma->timeout.initialized = true; 313 314 return 0; 315 } 316 317 /* 318 * Clean up timeout resources 319 */ 320 static void cdma_timeout_destroy(struct host1x_cdma *cdma) 321 { 322 if (cdma->timeout.initialized) 323 cancel_delayed_work(&cdma->timeout.wq); 324 325 cdma->timeout.initialized = false; 326 } 327 328 static const struct host1x_cdma_ops host1x_cdma_ops = { 329 .start = cdma_start, 330 .stop = cdma_stop, 331 .flush = cdma_flush, 332 333 .timeout_init = cdma_timeout_init, 334 .timeout_destroy = cdma_timeout_destroy, 335 .freeze = cdma_freeze, 336 .resume = cdma_resume, 337 .timeout_cpu_incr = cdma_timeout_cpu_incr, 338 }; 339 340 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { 341 .init = push_buffer_init, 342 }; 343