1 /* 2 * Tegra host1x Command DMA 3 * 4 * Copyright (c) 2010-2013, NVIDIA Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 20 #include <asm/cacheflush.h> 21 #include <linux/device.h> 22 #include <linux/dma-mapping.h> 23 #include <linux/host1x.h> 24 #include <linux/interrupt.h> 25 #include <linux/kernel.h> 26 #include <linux/kfifo.h> 27 #include <linux/slab.h> 28 #include <trace/events/host1x.h> 29 30 #include "cdma.h" 31 #include "channel.h" 32 #include "dev.h" 33 #include "debug.h" 34 #include "job.h" 35 36 /* 37 * push_buffer 38 * 39 * The push buffer is a circular array of words to be fetched by command DMA. 40 * Note that it works slightly differently to the sync queue; fence == pos 41 * means that the push buffer is full, not empty. 42 */ 43 44 #define HOST1X_PUSHBUFFER_SLOTS 512 45 46 /* 47 * Clean up push buffer resources 48 */ 49 static void host1x_pushbuffer_destroy(struct push_buffer *pb) 50 { 51 struct host1x_cdma *cdma = pb_to_cdma(pb); 52 struct host1x *host1x = cdma_to_host1x(cdma); 53 54 if (pb->phys != 0) 55 dma_free_wc(host1x->dev, pb->size_bytes + 4, pb->mapped, 56 pb->phys); 57 58 pb->mapped = NULL; 59 pb->phys = 0; 60 } 61 62 /* 63 * Init push buffer resources 64 */ 65 static int host1x_pushbuffer_init(struct push_buffer *pb) 66 { 67 struct host1x_cdma *cdma = pb_to_cdma(pb); 68 struct host1x *host1x = cdma_to_host1x(cdma); 69 70 pb->mapped = NULL; 71 pb->phys = 0; 72 pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8; 73 74 /* initialize buffer pointers */ 75 pb->fence = pb->size_bytes - 8; 76 pb->pos = 0; 77 78 /* allocate and map pushbuffer memory */ 79 pb->mapped = dma_alloc_wc(host1x->dev, pb->size_bytes + 4, &pb->phys, 80 GFP_KERNEL); 81 if (!pb->mapped) 82 goto fail; 83 84 host1x_hw_pushbuffer_init(host1x, pb); 85 86 return 0; 87 88 fail: 89 host1x_pushbuffer_destroy(pb); 90 return -ENOMEM; 91 } 92 93 /* 94 * Push two words to the push buffer 95 * Caller must ensure push buffer is not full 96 */ 97 static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2) 98 { 99 u32 *p = (u32 *)((void *)pb->mapped + pb->pos); 100 101 WARN_ON(pb->pos == pb->fence); 102 *(p++) = op1; 103 *(p++) = op2; 104 pb->pos = (pb->pos + 8) & (pb->size_bytes - 1); 105 } 106 107 /* 108 * Pop a number of two word slots from the push buffer 109 * Caller must ensure push buffer is not empty 110 */ 111 static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots) 112 { 113 /* Advance the next write position */ 114 pb->fence = (pb->fence + slots * 8) & (pb->size_bytes - 1); 115 } 116 117 /* 118 * Return the number of two word slots free in the push buffer 119 */ 120 static u32 host1x_pushbuffer_space(struct push_buffer *pb) 121 { 122 return ((pb->fence - pb->pos) & (pb->size_bytes - 1)) / 8; 123 } 124 125 /* 126 * Sleep (if necessary) until the requested event happens 127 * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. 128 * - Returns 1 129 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer 130 * - Return the amount of space (> 0) 131 * Must be called with the cdma lock held. 132 */ 133 unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, 134 enum cdma_event event) 135 { 136 for (;;) { 137 struct push_buffer *pb = &cdma->push_buffer; 138 unsigned int space; 139 140 switch (event) { 141 case CDMA_EVENT_SYNC_QUEUE_EMPTY: 142 space = list_empty(&cdma->sync_queue) ? 1 : 0; 143 break; 144 145 case CDMA_EVENT_PUSH_BUFFER_SPACE: 146 space = host1x_pushbuffer_space(pb); 147 break; 148 149 default: 150 WARN_ON(1); 151 return -EINVAL; 152 } 153 154 if (space) 155 return space; 156 157 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), 158 event); 159 160 /* If somebody has managed to already start waiting, yield */ 161 if (cdma->event != CDMA_EVENT_NONE) { 162 mutex_unlock(&cdma->lock); 163 schedule(); 164 mutex_lock(&cdma->lock); 165 continue; 166 } 167 168 cdma->event = event; 169 170 mutex_unlock(&cdma->lock); 171 down(&cdma->sem); 172 mutex_lock(&cdma->lock); 173 } 174 175 return 0; 176 } 177 178 /* 179 * Start timer that tracks the time spent by the job. 180 * Must be called with the cdma lock held. 181 */ 182 static void cdma_start_timer_locked(struct host1x_cdma *cdma, 183 struct host1x_job *job) 184 { 185 struct host1x *host = cdma_to_host1x(cdma); 186 187 if (cdma->timeout.client) { 188 /* timer already started */ 189 return; 190 } 191 192 cdma->timeout.client = job->client; 193 cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id); 194 cdma->timeout.syncpt_val = job->syncpt_end; 195 cdma->timeout.start_ktime = ktime_get(); 196 197 schedule_delayed_work(&cdma->timeout.wq, 198 msecs_to_jiffies(job->timeout)); 199 } 200 201 /* 202 * Stop timer when a buffer submission completes. 203 * Must be called with the cdma lock held. 204 */ 205 static void stop_cdma_timer_locked(struct host1x_cdma *cdma) 206 { 207 cancel_delayed_work(&cdma->timeout.wq); 208 cdma->timeout.client = 0; 209 } 210 211 /* 212 * For all sync queue entries that have already finished according to the 213 * current sync point registers: 214 * - unpin & unref their mems 215 * - pop their push buffer slots 216 * - remove them from the sync queue 217 * This is normally called from the host code's worker thread, but can be 218 * called manually if necessary. 219 * Must be called with the cdma lock held. 220 */ 221 static void update_cdma_locked(struct host1x_cdma *cdma) 222 { 223 bool signal = false; 224 struct host1x *host1x = cdma_to_host1x(cdma); 225 struct host1x_job *job, *n; 226 227 /* If CDMA is stopped, queue is cleared and we can return */ 228 if (!cdma->running) 229 return; 230 231 /* 232 * Walk the sync queue, reading the sync point registers as necessary, 233 * to consume as many sync queue entries as possible without blocking 234 */ 235 list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { 236 struct host1x_syncpt *sp = 237 host1x_syncpt_get(host1x, job->syncpt_id); 238 239 /* Check whether this syncpt has completed, and bail if not */ 240 if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) { 241 /* Start timer on next pending syncpt */ 242 if (job->timeout) 243 cdma_start_timer_locked(cdma, job); 244 245 break; 246 } 247 248 /* Cancel timeout, when a buffer completes */ 249 if (cdma->timeout.client) 250 stop_cdma_timer_locked(cdma); 251 252 /* Unpin the memory */ 253 host1x_job_unpin(job); 254 255 /* Pop push buffer slots */ 256 if (job->num_slots) { 257 struct push_buffer *pb = &cdma->push_buffer; 258 259 host1x_pushbuffer_pop(pb, job->num_slots); 260 261 if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) 262 signal = true; 263 } 264 265 list_del(&job->list); 266 host1x_job_put(job); 267 } 268 269 if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && 270 list_empty(&cdma->sync_queue)) 271 signal = true; 272 273 if (signal) { 274 cdma->event = CDMA_EVENT_NONE; 275 up(&cdma->sem); 276 } 277 } 278 279 void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, 280 struct device *dev) 281 { 282 struct host1x *host1x = cdma_to_host1x(cdma); 283 u32 restart_addr, syncpt_incrs, syncpt_val; 284 struct host1x_job *job = NULL; 285 286 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 287 288 dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", 289 __func__, syncpt_val); 290 291 /* 292 * Move the sync_queue read pointer to the first entry that hasn't 293 * completed based on the current HW syncpt value. It's likely there 294 * won't be any (i.e. we're still at the head), but covers the case 295 * where a syncpt incr happens just prior/during the teardown. 296 */ 297 298 dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", 299 __func__); 300 301 list_for_each_entry(job, &cdma->sync_queue, list) { 302 if (syncpt_val < job->syncpt_end) 303 break; 304 305 host1x_job_dump(dev, job); 306 } 307 308 /* 309 * Walk the sync_queue, first incrementing with the CPU syncpts that 310 * are partially executed (the first buffer) or fully skipped while 311 * still in the current context (slots are also NOP-ed). 312 * 313 * At the point contexts are interleaved, syncpt increments must be 314 * done inline with the pushbuffer from a GATHER buffer to maintain 315 * the order (slots are modified to be a GATHER of syncpt incrs). 316 * 317 * Note: save in restart_addr the location where the timed out buffer 318 * started in the PB, so we can start the refetch from there (with the 319 * modified NOP-ed PB slots). This lets things appear to have completed 320 * properly for this buffer and resources are freed. 321 */ 322 323 dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n", 324 __func__); 325 326 if (!list_empty(&cdma->sync_queue)) 327 restart_addr = job->first_get; 328 else 329 restart_addr = cdma->last_pos; 330 331 /* do CPU increments as long as this context continues */ 332 list_for_each_entry_from(job, &cdma->sync_queue, list) { 333 /* different context, gets us out of this loop */ 334 if (job->client != cdma->timeout.client) 335 break; 336 337 /* won't need a timeout when replayed */ 338 job->timeout = 0; 339 340 syncpt_incrs = job->syncpt_end - syncpt_val; 341 dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); 342 343 host1x_job_dump(dev, job); 344 345 /* safe to use CPU to incr syncpts */ 346 host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, 347 syncpt_incrs, job->syncpt_end, 348 job->num_slots); 349 350 syncpt_val += syncpt_incrs; 351 } 352 353 /* 354 * The following sumbits from the same client may be dependent on the 355 * failed submit and therefore they may fail. Force a small timeout 356 * to make the queue cleanup faster. 357 */ 358 359 list_for_each_entry_from(job, &cdma->sync_queue, list) 360 if (job->client == cdma->timeout.client) 361 job->timeout = min_t(unsigned int, job->timeout, 500); 362 363 dev_dbg(dev, "%s: finished sync_queue modification\n", __func__); 364 365 /* roll back DMAGET and start up channel again */ 366 host1x_hw_cdma_resume(host1x, cdma, restart_addr); 367 } 368 369 /* 370 * Create a cdma 371 */ 372 int host1x_cdma_init(struct host1x_cdma *cdma) 373 { 374 int err; 375 376 mutex_init(&cdma->lock); 377 sema_init(&cdma->sem, 0); 378 379 INIT_LIST_HEAD(&cdma->sync_queue); 380 381 cdma->event = CDMA_EVENT_NONE; 382 cdma->running = false; 383 cdma->torndown = false; 384 385 err = host1x_pushbuffer_init(&cdma->push_buffer); 386 if (err) 387 return err; 388 389 return 0; 390 } 391 392 /* 393 * Destroy a cdma 394 */ 395 int host1x_cdma_deinit(struct host1x_cdma *cdma) 396 { 397 struct push_buffer *pb = &cdma->push_buffer; 398 struct host1x *host1x = cdma_to_host1x(cdma); 399 400 if (cdma->running) { 401 pr_warn("%s: CDMA still running\n", __func__); 402 return -EBUSY; 403 } 404 405 host1x_pushbuffer_destroy(pb); 406 host1x_hw_cdma_timeout_destroy(host1x, cdma); 407 408 return 0; 409 } 410 411 /* 412 * Begin a cdma submit 413 */ 414 int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job) 415 { 416 struct host1x *host1x = cdma_to_host1x(cdma); 417 418 mutex_lock(&cdma->lock); 419 420 if (job->timeout) { 421 /* init state on first submit with timeout value */ 422 if (!cdma->timeout.initialized) { 423 int err; 424 425 err = host1x_hw_cdma_timeout_init(host1x, cdma, 426 job->syncpt_id); 427 if (err) { 428 mutex_unlock(&cdma->lock); 429 return err; 430 } 431 } 432 } 433 434 if (!cdma->running) 435 host1x_hw_cdma_start(host1x, cdma); 436 437 cdma->slots_free = 0; 438 cdma->slots_used = 0; 439 cdma->first_get = cdma->push_buffer.pos; 440 441 trace_host1x_cdma_begin(dev_name(job->channel->dev)); 442 return 0; 443 } 444 445 /* 446 * Push two words into a push buffer slot 447 * Blocks as necessary if the push buffer is full. 448 */ 449 void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2) 450 { 451 struct host1x *host1x = cdma_to_host1x(cdma); 452 struct push_buffer *pb = &cdma->push_buffer; 453 u32 slots_free = cdma->slots_free; 454 455 if (host1x_debug_trace_cmdbuf) 456 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev), 457 op1, op2); 458 459 if (slots_free == 0) { 460 host1x_hw_cdma_flush(host1x, cdma); 461 slots_free = host1x_cdma_wait_locked(cdma, 462 CDMA_EVENT_PUSH_BUFFER_SPACE); 463 } 464 465 cdma->slots_free = slots_free - 1; 466 cdma->slots_used++; 467 host1x_pushbuffer_push(pb, op1, op2); 468 } 469 470 /* 471 * End a cdma submit 472 * Kick off DMA, add job to the sync queue, and a number of slots to be freed 473 * from the pushbuffer. The handles for a submit must all be pinned at the same 474 * time, but they can be unpinned in smaller chunks. 475 */ 476 void host1x_cdma_end(struct host1x_cdma *cdma, 477 struct host1x_job *job) 478 { 479 struct host1x *host1x = cdma_to_host1x(cdma); 480 bool idle = list_empty(&cdma->sync_queue); 481 482 host1x_hw_cdma_flush(host1x, cdma); 483 484 job->first_get = cdma->first_get; 485 job->num_slots = cdma->slots_used; 486 host1x_job_get(job); 487 list_add_tail(&job->list, &cdma->sync_queue); 488 489 /* start timer on idle -> active transitions */ 490 if (job->timeout && idle) 491 cdma_start_timer_locked(cdma, job); 492 493 trace_host1x_cdma_end(dev_name(job->channel->dev)); 494 mutex_unlock(&cdma->lock); 495 } 496 497 /* 498 * Update cdma state according to current sync point values 499 */ 500 void host1x_cdma_update(struct host1x_cdma *cdma) 501 { 502 mutex_lock(&cdma->lock); 503 update_cdma_locked(cdma); 504 mutex_unlock(&cdma->lock); 505 } 506