1 /* 2 * Tegra host1x Command DMA 3 * 4 * Copyright (c) 2010-2013, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #include <linux/slab.h> 20 #include <linux/scatterlist.h> 21 #include <linux/dma-mapping.h> 22 23 #include "../cdma.h" 24 #include "../channel.h" 25 #include "../dev.h" 26 #include "../debug.h" 27 28 /* 29 * Put the restart at the end of pushbuffer memor 30 */ 31 static void push_buffer_init(struct push_buffer *pb) 32 { 33 *(pb->mapped + (pb->size_bytes >> 2)) = host1x_opcode_restart(0); 34 } 35 36 /* 37 * Increment timedout buffer's syncpt via CPU. 38 */ 39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, 40 u32 syncpt_incrs, u32 syncval, u32 nr_slots) 41 { 42 struct host1x *host1x = cdma_to_host1x(cdma); 43 struct push_buffer *pb = &cdma->push_buffer; 44 u32 i; 45 46 for (i = 0; i < syncpt_incrs; i++) 47 host1x_syncpt_incr(cdma->timeout.syncpt); 48 49 /* after CPU incr, ensure shadow is up to date */ 50 host1x_syncpt_load(cdma->timeout.syncpt); 51 52 /* NOP all the PB slots */ 53 while (nr_slots--) { 54 u32 *p = (u32 *)((u32)pb->mapped + getptr); 55 *(p++) = HOST1X_OPCODE_NOP; 56 *(p++) = HOST1X_OPCODE_NOP; 57 dev_dbg(host1x->dev, "%s: NOP at %#llx\n", __func__, 58 (u64)pb->phys + getptr); 59 getptr = (getptr + 8) & (pb->size_bytes - 1); 60 } 61 wmb(); 62 } 63 64 /* 65 * Start channel DMA 66 */ 67 static void cdma_start(struct host1x_cdma *cdma) 68 { 69 struct host1x_channel *ch = cdma_to_channel(cdma); 70 71 if (cdma->running) 72 return; 73 74 cdma->last_pos = cdma->push_buffer.pos; 75 76 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 77 HOST1X_CHANNEL_DMACTRL); 78 79 /* set base, put and end pointer */ 80 host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART); 81 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 82 host1x_ch_writel(ch, cdma->push_buffer.phys + 83 cdma->push_buffer.size_bytes + 4, 84 HOST1X_CHANNEL_DMAEND); 85 86 /* reset GET */ 87 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 88 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 89 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 90 HOST1X_CHANNEL_DMACTRL); 91 92 /* start the command DMA */ 93 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 94 95 cdma->running = true; 96 } 97 98 /* 99 * Similar to cdma_start(), but rather than starting from an idle 100 * state (where DMA GET is set to DMA PUT), on a timeout we restore 101 * DMA GET from an explicit value (so DMA may again be pending). 102 */ 103 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) 104 { 105 struct host1x *host1x = cdma_to_host1x(cdma); 106 struct host1x_channel *ch = cdma_to_channel(cdma); 107 108 if (cdma->running) 109 return; 110 111 cdma->last_pos = cdma->push_buffer.pos; 112 113 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 114 HOST1X_CHANNEL_DMACTRL); 115 116 /* set base, end pointer (all of memory) */ 117 host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART); 118 host1x_ch_writel(ch, cdma->push_buffer.phys + 119 cdma->push_buffer.size_bytes, 120 HOST1X_CHANNEL_DMAEND); 121 122 /* set GET, by loading the value in PUT (then reset GET) */ 123 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT); 124 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP | 125 HOST1X_CHANNEL_DMACTRL_DMAGETRST | 126 HOST1X_CHANNEL_DMACTRL_DMAINITGET, 127 HOST1X_CHANNEL_DMACTRL); 128 129 dev_dbg(host1x->dev, 130 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__, 131 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 132 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 133 cdma->last_pos); 134 135 /* deassert GET reset and set PUT */ 136 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 137 HOST1X_CHANNEL_DMACTRL); 138 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); 139 140 /* start the command DMA */ 141 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL); 142 143 cdma->running = true; 144 } 145 146 /* 147 * Kick channel DMA into action by writing its PUT offset (if it has changed) 148 */ 149 static void cdma_flush(struct host1x_cdma *cdma) 150 { 151 struct host1x_channel *ch = cdma_to_channel(cdma); 152 153 if (cdma->push_buffer.pos != cdma->last_pos) { 154 host1x_ch_writel(ch, cdma->push_buffer.pos, 155 HOST1X_CHANNEL_DMAPUT); 156 cdma->last_pos = cdma->push_buffer.pos; 157 } 158 } 159 160 static void cdma_stop(struct host1x_cdma *cdma) 161 { 162 struct host1x_channel *ch = cdma_to_channel(cdma); 163 164 mutex_lock(&cdma->lock); 165 if (cdma->running) { 166 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY); 167 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 168 HOST1X_CHANNEL_DMACTRL); 169 cdma->running = false; 170 } 171 mutex_unlock(&cdma->lock); 172 } 173 174 /* 175 * Stops both channel's command processor and CDMA immediately. 176 * Also, tears down the channel and resets corresponding module. 177 */ 178 static void cdma_freeze(struct host1x_cdma *cdma) 179 { 180 struct host1x *host = cdma_to_host1x(cdma); 181 struct host1x_channel *ch = cdma_to_channel(cdma); 182 u32 cmdproc_stop; 183 184 if (cdma->torndown && !cdma->running) { 185 dev_warn(host->dev, "Already torn down\n"); 186 return; 187 } 188 189 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id); 190 191 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP); 192 cmdproc_stop |= BIT(ch->id); 193 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 194 195 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", 196 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET), 197 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT), 198 cdma->last_pos); 199 200 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP, 201 HOST1X_CHANNEL_DMACTRL); 202 203 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN); 204 205 cdma->running = false; 206 cdma->torndown = true; 207 } 208 209 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr) 210 { 211 struct host1x *host1x = cdma_to_host1x(cdma); 212 struct host1x_channel *ch = cdma_to_channel(cdma); 213 u32 cmdproc_stop; 214 215 dev_dbg(host1x->dev, 216 "resuming channel (id %d, DMAGET restart = 0x%x)\n", 217 ch->id, getptr); 218 219 cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP); 220 cmdproc_stop &= ~(BIT(ch->id)); 221 host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 222 223 cdma->torndown = false; 224 cdma_timeout_restart(cdma, getptr); 225 } 226 227 /* 228 * If this timeout fires, it indicates the current sync_queue entry has 229 * exceeded its TTL and the userctx should be timed out and remaining 230 * submits already issued cleaned up (future submits return an error). 231 */ 232 static void cdma_timeout_handler(struct work_struct *work) 233 { 234 struct host1x_cdma *cdma; 235 struct host1x *host1x; 236 struct host1x_channel *ch; 237 238 u32 syncpt_val; 239 240 u32 prev_cmdproc, cmdproc_stop; 241 242 cdma = container_of(to_delayed_work(work), struct host1x_cdma, 243 timeout.wq); 244 host1x = cdma_to_host1x(cdma); 245 ch = cdma_to_channel(cdma); 246 247 host1x_debug_dump(cdma_to_host1x(cdma)); 248 249 mutex_lock(&cdma->lock); 250 251 if (!cdma->timeout.client) { 252 dev_dbg(host1x->dev, 253 "cdma_timeout: expired, but has no clientid\n"); 254 mutex_unlock(&cdma->lock); 255 return; 256 } 257 258 /* stop processing to get a clean snapshot */ 259 prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP); 260 cmdproc_stop = prev_cmdproc | BIT(ch->id); 261 host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP); 262 263 dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n", 264 prev_cmdproc, cmdproc_stop); 265 266 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 267 268 /* has buffer actually completed? */ 269 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) { 270 dev_dbg(host1x->dev, 271 "cdma_timeout: expired, but buffer had completed\n"); 272 /* restore */ 273 cmdproc_stop = prev_cmdproc & ~(BIT(ch->id)); 274 host1x_sync_writel(host1x, cmdproc_stop, 275 HOST1X_SYNC_CMDPROC_STOP); 276 mutex_unlock(&cdma->lock); 277 return; 278 } 279 280 dev_warn(host1x->dev, "%s: timeout: %d (%s), HW thresh %d, done %d\n", 281 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name, 282 syncpt_val, cdma->timeout.syncpt_val); 283 284 /* stop HW, resetting channel/module */ 285 host1x_hw_cdma_freeze(host1x, cdma); 286 287 host1x_cdma_update_sync_queue(cdma, ch->dev); 288 mutex_unlock(&cdma->lock); 289 } 290 291 /* 292 * Init timeout resources 293 */ 294 static int cdma_timeout_init(struct host1x_cdma *cdma, u32 syncpt_id) 295 { 296 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler); 297 cdma->timeout.initialized = true; 298 299 return 0; 300 } 301 302 /* 303 * Clean up timeout resources 304 */ 305 static void cdma_timeout_destroy(struct host1x_cdma *cdma) 306 { 307 if (cdma->timeout.initialized) 308 cancel_delayed_work(&cdma->timeout.wq); 309 cdma->timeout.initialized = false; 310 } 311 312 static const struct host1x_cdma_ops host1x_cdma_ops = { 313 .start = cdma_start, 314 .stop = cdma_stop, 315 .flush = cdma_flush, 316 317 .timeout_init = cdma_timeout_init, 318 .timeout_destroy = cdma_timeout_destroy, 319 .freeze = cdma_freeze, 320 .resume = cdma_resume, 321 .timeout_cpu_incr = cdma_timeout_cpu_incr, 322 }; 323 324 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = { 325 .init = push_buffer_init, 326 }; 327