1 /* 2 * Tegra host1x Command DMA 3 * 4 * Copyright (c) 2010-2013, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/slab.h> 20 #include <linux/scatterlist.h> 21 #include <linux/dma-mapping.h> 22 23 #include "../cdma.h" 24 #include "../channel.h" 25 #include "../dev.h" 26 #include "../debug.h" 27 28 /* 29 * Put the restart at the end of pushbuffer memory 30 */ 31 static void push_buffer_init(struct push_buffer *pb) 32 { 33 *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0); 34 } 35 36 /* 37 * Increment timedout buffer's syncpt via CPU. 38 */ 39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, 40 u32 syncpt_incrs, u32 syncval, u32 nr_slots) 41 { 42 struct host1x *host1x = cdma_to_host1x(cdma); 43 struct push_buffer *pb = &cdma->push_buffer; 44 unsigned int i; 45 46 for (i = 0; i < syncpt_incrs; i++) 47 host1x_syncpt_incr(cdma->timeout.syncpt); 48 49 /* after CPU incr, ensure shadow is up to date */ 50 host1x_syncpt_load(cdma->timeout.syncpt); 51 52 /* NOP all the PB slots */ 53 while (nr_slots--) { 54 u32 *p = (u32 *)(pb->mapped + getptr); 55 *(p++) = HOST1X_OPCODE_NOP; 56 *(p++) = HOST1X_OPCODE_NOP; 57 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__, 58 &pb->dma, getptr); 59 getptr = (getptr + 8) & (pb->size - 1); 60 } 61 62 wmb(); 63 } 64 65 /* 66 * Start channel DMA 67 */ 68 static void cdma_start(struct host1x_cdma *cdma) 69 { 70 struct host1x_channel *ch = cdma_to_channel(cdma); 71 72 if (cdma->running) 73 return; 74 75 cdma->last_pos = cdma->push_buffer.pos; 76 77 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 78 HOST1X_CHANNEL_DMACTRL); 79 80 /* set base, put and end pointer */ 81 host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART); 82 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 83 host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size + 4, 84 HOST1X_CHANNEL_DMAEND); 85 86 /* reset GET */ 87 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 88 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 89 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 90 HOST1X_CHANNEL_DMACTRL); 91 92 /* start the command DMA */ 93 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 94 95 cdma->running = true; 96 } 97 98 /* 99 * Similar to cdma_start(), but rather than starting from an idle 100 * state (where DMA GET is set to DMA PUT), on a timeout we restore 101 * DMA GET from an explicit value (so DMA may again be pending). 102 */ 103 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) 104 { 105 struct host1x *host1x = cdma_to_host1x(cdma); 106 struct host1x_channel *ch = cdma_to_channel(cdma); 107 108 if (cdma->running) 109 return; 110 111 cdma->last_pos = cdma->push_buffer.pos; 112 113 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 114 HOST1X_CHANNEL_DMACTRL); 115 116 /* set base, end pointer (all of memory) */ 117 host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART); 118 host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size, 119 HOST1X_CHANNEL_DMAEND); 120 121 /* set GET, by loading the value in PUT (then reset GET) */ 122 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT); 123 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 124 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 125 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 126 HOST1X_CHANNEL_DMACTRL); 127 128 dev_dbg(host1x->dev, 129 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__, 130 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 131 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 132 cdma->last_pos); 133 134 /* deassert GET reset and set PUT */ 135 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 136 HOST1X_CHANNEL_DMACTRL); 137 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 138 139 /* start the command DMA */ 140 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 141 142 cdma->running = true; 143 } 144 145 /* 146 * Kick channel DMA into action by writing its PUT offset (if it has changed) 147 */ 148 static void cdma_flush(struct host1x_cdma *cdma) 149 { 150 struct host1x_channel *ch = cdma_to_channel(cdma); 151 152 if (cdma->push_buffer.pos != cdma->last_pos) { 153 host1x_ch_writel(ch, cdma->push_buffer.pos, 154 HOST1X_CHANNEL_DMAPUT); 155 cdma->last_pos = cdma->push_buffer.pos; 156 } 157 } 158 159 static void cdma_stop(struct host1x_cdma *cdma) 160 { 161 struct host1x_channel *ch = cdma_to_channel(cdma); 162 163 mutex_lock(&cdma->lock); 164 165 if (cdma->running) { 166 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY); 167 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 168 HOST1X_CHANNEL_DMACTRL); 169 cdma->running = false; 170 } 171 172 mutex_unlock(&cdma->lock); 173 } 174 175 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch, 176 bool stop) 177 { 178 #if HOST1X_HW >= 6 179 host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP); 180 #else 181 u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP); 182 if (stop) 183 cmdproc_stop |= BIT(ch->id); 184 else 185 cmdproc_stop &= ~BIT(ch->id); 186 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 187 #endif 188 } 189 190 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch) 191 { 192 #if HOST1X_HW >= 6 193 host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN); 194 #else 195 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN); 196 #endif 197 } 198 199 /* 200 * Stops both channel's command processor and CDMA immediately. 201 * Also, tears down the channel and resets corresponding module. 202 */ 203 static void cdma_freeze(struct host1x_cdma *cdma) 204 { 205 struct host1x *host = cdma_to_host1x(cdma); 206 struct host1x_channel *ch = cdma_to_channel(cdma); 207 208 if (cdma->torndown && !cdma->running) { 209 dev_warn(host->dev, "Already torn down\n"); 210 return; 211 } 212 213 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id); 214 215 cdma_hw_cmdproc_stop(host, ch, true); 216 217 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", 218 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 219 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 220 cdma->last_pos); 221 222 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 223 HOST1X_CHANNEL_DMACTRL); 224 225 cdma_hw_teardown(host, ch); 226 227 cdma->running = false; 228 cdma->torndown = true; 229 } 230 231 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) 232 { 233 struct host1x *host1x = cdma_to_host1x(cdma); 234 struct host1x_channel *ch = cdma_to_channel(cdma); 235 236 dev_dbg(host1x->dev, 237 "resuming channel (id %u, DMAGET restart = 0x%x)\n", 238 ch->id, getptr); 239 240 cdma_hw_cmdproc_stop(host1x, ch, false); 241 242 cdma->torndown = false; 243 cdma_timeout_restart(cdma, getptr); 244 } 245 246 /* 247 * If this timeout fires, it indicates the current sync_queue entry has 248 * exceeded its TTL and the userctx should be timed out and remaining 249 * submits already issued cleaned up (future submits return an error). 250 */ 251 static void cdma_timeout_handler(struct work_struct *work) 252 { 253 u32 syncpt_val; 254 struct host1x_cdma *cdma; 255 struct host1x *host1x; 256 struct host1x_channel *ch; 257 258 cdma = container_of(to_delayed_work(work), struct host1x_cdma, 259 timeout.wq); 260 host1x = cdma_to_host1x(cdma); 261 ch = cdma_to_channel(cdma); 262 263 host1x_debug_dump(cdma_to_host1x(cdma)); 264 265 mutex_lock(&cdma->lock); 266 267 if (!cdma->timeout.client) { 268 dev_dbg(host1x->dev, 269 "cdma_timeout: expired, but has no clientid\n"); 270 mutex_unlock(&cdma->lock); 271 return; 272 } 273 274 /* stop processing to get a clean snapshot */ 275 cdma_hw_cmdproc_stop(host1x, ch, true); 276 277 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 278 279 /* has buffer actually completed? */ 280 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { 281 dev_dbg(host1x->dev, 282 "cdma_timeout: expired, but buffer had completed\n"); 283 /* restore */ 284 cdma_hw_cmdproc_stop(host1x, ch, false); 285 mutex_unlock(&cdma->lock); 286 return; 287 } 288 289 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n", 290 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, 291 syncpt_val, cdma->timeout.syncpt_val); 292 293 /* stop HW, resetting channel/module */ 294 host1x_hw_cdma_freeze(host1x, cdma); 295 296 host1x_cdma_update_sync_queue(cdma, ch->dev); 297 mutex_unlock(&cdma->lock); 298 } 299 300 /* 301 * Init timeout resources 302 */ 303 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt) 304 { 305 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); 306 cdma->timeout.initialized = true; 307 308 return 0; 309 } 310 311 /* 312 * Clean up timeout resources 313 */ 314 static void cdma_timeout_destroy(struct host1x_cdma *cdma) 315 { 316 if (cdma->timeout.initialized) 317 cancel_delayed_work(&cdma->timeout.wq); 318 319 cdma->timeout.initialized = false; 320 } 321 322 static const struct host1x_cdma_ops host1x_cdma_ops = { 323 .start = cdma_start, 324 .stop = cdma_stop, 325 .flush = cdma_flush, 326 327 .timeout_init = cdma_timeout_init, 328 .timeout_destroy = cdma_timeout_destroy, 329 .freeze = cdma_freeze, 330 .resume = cdma_resume, 331 .timeout_cpu_incr = cdma_timeout_cpu_incr, 332 }; 333 334 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { 335 .init = push_buffer_init, 336 }; 337