1 /* 2 * Tegra host1x Command DMA 3 * 4 * Copyright (c) 2010-2013, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/slab.h> 20 #include <linux/scatterlist.h> 21 #include <linux/dma-mapping.h> 22 23 #include "../cdma.h" 24 #include "../channel.h" 25 #include "../dev.h" 26 #include "../debug.h" 27 28 /* 29 * Put the restart at the end of pushbuffer memory 30 */ 31 static void push_buffer_init(struct push_buffer *pb) 32 { 33 *(u32 *)(pb->mapped + pb->size_bytes) = host1x_opcode_restart(0); 34 } 35 36 /* 37 * Increment timedout buffer's syncpt via CPU. 38 */ 39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, 40 u32 syncpt_incrs, u32 syncval, u32 nr_slots) 41 { 42 struct host1x *host1x = cdma_to_host1x(cdma); 43 struct push_buffer *pb = &cdma->push_buffer; 44 unsigned int i; 45 46 for (i = 0; i < syncpt_incrs; i++) 47 host1x_syncpt_incr(cdma->timeout.syncpt); 48 49 /* after CPU incr, ensure shadow is up to date */ 50 host1x_syncpt_load(cdma->timeout.syncpt); 51 52 /* NOP all the PB slots */ 53 while (nr_slots--) { 54 u32 *p = (u32 *)(pb->mapped + getptr); 55 *(p++) = HOST1X_OPCODE_NOP; 56 *(p++) = HOST1X_OPCODE_NOP; 57 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__, 58 &pb->phys, getptr); 59 getptr = (getptr + 8) & (pb->size_bytes - 1); 60 } 61 62 wmb(); 63 } 64 65 /* 66 * Start channel DMA 67 */ 68 static void cdma_start(struct host1x_cdma *cdma) 69 { 70 struct host1x_channel *ch = cdma_to_channel(cdma); 71 72 if (cdma->running) 73 return; 74 75 cdma->last_pos = cdma->push_buffer.pos; 76 77 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 78 HOST1X_CHANNEL_DMACTRL); 79 80 /* set base, put and end pointer */ 81 host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART); 82 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 83 host1x_ch_writel(ch, cdma->push_buffer.phys + 84 cdma->push_buffer.size_bytes + 4, 85 HOST1X_CHANNEL_DMAEND); 86 87 /* reset GET */ 88 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 89 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 90 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 91 HOST1X_CHANNEL_DMACTRL); 92 93 /* start the command DMA */ 94 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 95 96 cdma->running = true; 97 } 98 99 /* 100 * Similar to cdma_start(), but rather than starting from an idle 101 * state (where DMA GET is set to DMA PUT), on a timeout we restore 102 * DMA GET from an explicit value (so DMA may again be pending). 103 */ 104 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) 105 { 106 struct host1x *host1x = cdma_to_host1x(cdma); 107 struct host1x_channel *ch = cdma_to_channel(cdma); 108 109 if (cdma->running) 110 return; 111 112 cdma->last_pos = cdma->push_buffer.pos; 113 114 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 115 HOST1X_CHANNEL_DMACTRL); 116 117 /* set base, end pointer (all of memory) */ 118 host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART); 119 host1x_ch_writel(ch, cdma->push_buffer.phys + 120 cdma->push_buffer.size_bytes, 121 HOST1X_CHANNEL_DMAEND); 122 123 /* set GET, by loading the value in PUT (then reset GET) */ 124 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT); 125 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 126 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 127 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 128 HOST1X_CHANNEL_DMACTRL); 129 130 dev_dbg(host1x->dev, 131 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__, 132 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 133 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 134 cdma->last_pos); 135 136 /* deassert GET reset and set PUT */ 137 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 138 HOST1X_CHANNEL_DMACTRL); 139 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 140 141 /* start the command DMA */ 142 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 143 144 cdma->running = true; 145 } 146 147 /* 148 * Kick channel DMA into action by writing its PUT offset (if it has changed) 149 */ 150 static void cdma_flush(struct host1x_cdma *cdma) 151 { 152 struct host1x_channel *ch = cdma_to_channel(cdma); 153 154 if (cdma->push_buffer.pos != cdma->last_pos) { 155 host1x_ch_writel(ch, cdma->push_buffer.pos, 156 HOST1X_CHANNEL_DMAPUT); 157 cdma->last_pos = cdma->push_buffer.pos; 158 } 159 } 160 161 static void cdma_stop(struct host1x_cdma *cdma) 162 { 163 struct host1x_channel *ch = cdma_to_channel(cdma); 164 165 mutex_lock(&cdma->lock); 166 167 if (cdma->running) { 168 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY); 169 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 170 HOST1X_CHANNEL_DMACTRL); 171 cdma->running = false; 172 } 173 174 mutex_unlock(&cdma->lock); 175 } 176 177 /* 178 * Stops both channel's command processor and CDMA immediately. 179 * Also, tears down the channel and resets corresponding module. 180 */ 181 static void cdma_freeze(struct host1x_cdma *cdma) 182 { 183 struct host1x *host = cdma_to_host1x(cdma); 184 struct host1x_channel *ch = cdma_to_channel(cdma); 185 u32 cmdproc_stop; 186 187 if (cdma->torndown && !cdma->running) { 188 dev_warn(host->dev, "Already torn down\n"); 189 return; 190 } 191 192 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id); 193 194 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP); 195 cmdproc_stop |= BIT(ch->id); 196 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 197 198 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", 199 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 200 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 201 cdma->last_pos); 202 203 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 204 HOST1X_CHANNEL_DMACTRL); 205 206 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN); 207 208 cdma->running = false; 209 cdma->torndown = true; 210 } 211 212 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) 213 { 214 struct host1x *host1x = cdma_to_host1x(cdma); 215 struct host1x_channel *ch = cdma_to_channel(cdma); 216 u32 cmdproc_stop; 217 218 dev_dbg(host1x->dev, 219 "resuming channel (id %u, DMAGET restart = 0x%x)\n", 220 ch->id, getptr); 221 222 cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP); 223 cmdproc_stop &= ~BIT(ch->id); 224 host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 225 226 cdma->torndown = false; 227 cdma_timeout_restart(cdma, getptr); 228 } 229 230 /* 231 * If this timeout fires, it indicates the current sync_queue entry has 232 * exceeded its TTL and the userctx should be timed out and remaining 233 * submits already issued cleaned up (future submits return an error). 234 */ 235 static void cdma_timeout_handler(struct work_struct *work) 236 { 237 u32 prev_cmdproc, cmdproc_stop, syncpt_val; 238 struct host1x_cdma *cdma; 239 struct host1x *host1x; 240 struct host1x_channel *ch; 241 242 cdma = container_of(to_delayed_work(work), struct host1x_cdma, 243 timeout.wq); 244 host1x = cdma_to_host1x(cdma); 245 ch = cdma_to_channel(cdma); 246 247 host1x_debug_dump(cdma_to_host1x(cdma)); 248 249 mutex_lock(&cdma->lock); 250 251 if (!cdma->timeout.client) { 252 dev_dbg(host1x->dev, 253 "cdma_timeout: expired, but has no clientid\n"); 254 mutex_unlock(&cdma->lock); 255 return; 256 } 257 258 /* stop processing to get a clean snapshot */ 259 prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP); 260 cmdproc_stop = prev_cmdproc | BIT(ch->id); 261 host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 262 263 dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n", 264 prev_cmdproc, cmdproc_stop); 265 266 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 267 268 /* has buffer actually completed? */ 269 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { 270 dev_dbg(host1x->dev, 271 "cdma_timeout: expired, but buffer had completed\n"); 272 /* restore */ 273 cmdproc_stop = prev_cmdproc & ~(BIT(ch->id)); 274 host1x_sync_writel(host1x, cmdproc_stop, 275 HOST1X_SYNC_CMDPROC_STOP); 276 mutex_unlock(&cdma->lock); 277 return; 278 } 279 280 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n", 281 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, 282 syncpt_val, cdma->timeout.syncpt_val); 283 284 /* stop HW, resetting channel/module */ 285 host1x_hw_cdma_freeze(host1x, cdma); 286 287 host1x_cdma_update_sync_queue(cdma, ch->dev); 288 mutex_unlock(&cdma->lock); 289 } 290 291 /* 292 * Init timeout resources 293 */ 294 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt) 295 { 296 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); 297 cdma->timeout.initialized = true; 298 299 return 0; 300 } 301 302 /* 303 * Clean up timeout resources 304 */ 305 static void cdma_timeout_destroy(struct host1x_cdma *cdma) 306 { 307 if (cdma->timeout.initialized) 308 cancel_delayed_work(&cdma->timeout.wq); 309 310 cdma->timeout.initialized = false; 311 } 312 313 static const struct host1x_cdma_ops host1x_cdma_ops = { 314 .start = cdma_start, 315 .stop = cdma_stop, 316 .flush = cdma_flush, 317 318 .timeout_init = cdma_timeout_init, 319 .timeout_destroy = cdma_timeout_destroy, 320 .freeze = cdma_freeze, 321 .resume = cdma_resume, 322 .timeout_cpu_incr = cdma_timeout_cpu_incr, 323 }; 324 325 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { 326 .init = push_buffer_init, 327 }; 328