16579324aSTerje Bergstrom /* 26579324aSTerje Bergstrom * Tegra host1x Command DMA 36579324aSTerje Bergstrom * 46579324aSTerje Bergstrom * Copyright (c) 2010-2013, NVIDIA Corporation. 56579324aSTerje Bergstrom * 66579324aSTerje Bergstrom * This program is free software; you can redistribute it and/or modify it 76579324aSTerje Bergstrom * under the terms and conditions of the GNU General Public License, 86579324aSTerje Bergstrom * version 2, as published by the Free Software Foundation. 96579324aSTerje Bergstrom * 106579324aSTerje Bergstrom * This program is distributed in the hope it will be useful, but WITHOUT 116579324aSTerje Bergstrom * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 126579324aSTerje Bergstrom * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 136579324aSTerje Bergstrom * more details. 146579324aSTerje Bergstrom * 156579324aSTerje Bergstrom * You should have received a copy of the GNU General Public License 166579324aSTerje Bergstrom * along with this program. If not, see <http://www.gnu.org/licenses/>. 176579324aSTerje Bergstrom */ 186579324aSTerje Bergstrom 196579324aSTerje Bergstrom 206579324aSTerje Bergstrom #include <asm/cacheflush.h> 216579324aSTerje Bergstrom #include <linux/device.h> 226579324aSTerje Bergstrom #include <linux/dma-mapping.h> 236579324aSTerje Bergstrom #include <linux/interrupt.h> 246579324aSTerje Bergstrom #include <linux/kernel.h> 256579324aSTerje Bergstrom #include <linux/kfifo.h> 266579324aSTerje Bergstrom #include <linux/slab.h> 276579324aSTerje Bergstrom #include <trace/events/host1x.h> 286579324aSTerje Bergstrom 296579324aSTerje Bergstrom #include "cdma.h" 306579324aSTerje Bergstrom #include "channel.h" 316579324aSTerje Bergstrom #include "dev.h" 326579324aSTerje Bergstrom #include "debug.h" 336579324aSTerje Bergstrom #include "host1x_bo.h" 346579324aSTerje Bergstrom #include "job.h" 356579324aSTerje Bergstrom 366579324aSTerje Bergstrom /* 376579324aSTerje Bergstrom * push_buffer 386579324aSTerje Bergstrom * 396579324aSTerje Bergstrom * The push buffer is a circular array of words to be fetched by command DMA. 406579324aSTerje Bergstrom * Note that it works slightly differently to the sync queue; fence == pos 416579324aSTerje Bergstrom * means that the push buffer is full, not empty. 426579324aSTerje Bergstrom */ 436579324aSTerje Bergstrom 446579324aSTerje Bergstrom #define HOST1X_PUSHBUFFER_SLOTS 512 456579324aSTerje Bergstrom 466579324aSTerje Bergstrom /* 476579324aSTerje Bergstrom * Clean up push buffer resources 486579324aSTerje Bergstrom */ 496579324aSTerje Bergstrom static void host1x_pushbuffer_destroy(struct push_buffer *pb) 506579324aSTerje Bergstrom { 516579324aSTerje Bergstrom struct host1x_cdma *cdma = pb_to_cdma(pb); 526579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 536579324aSTerje Bergstrom 546579324aSTerje Bergstrom if (pb->phys != 0) 556579324aSTerje Bergstrom dma_free_writecombine(host1x->dev, pb->size_bytes + 4, 566579324aSTerje Bergstrom pb->mapped, pb->phys); 576579324aSTerje Bergstrom 586579324aSTerje Bergstrom pb->mapped = NULL; 596579324aSTerje Bergstrom pb->phys = 0; 606579324aSTerje Bergstrom } 616579324aSTerje Bergstrom 626579324aSTerje Bergstrom /* 636579324aSTerje Bergstrom * Init push buffer resources 646579324aSTerje Bergstrom */ 656579324aSTerje Bergstrom static int host1x_pushbuffer_init(struct push_buffer *pb) 666579324aSTerje Bergstrom { 676579324aSTerje Bergstrom struct host1x_cdma *cdma = pb_to_cdma(pb); 686579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 696579324aSTerje Bergstrom 706579324aSTerje Bergstrom pb->mapped = NULL; 716579324aSTerje Bergstrom pb->phys = 0; 726579324aSTerje Bergstrom pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8; 736579324aSTerje Bergstrom 746579324aSTerje Bergstrom /* initialize buffer pointers */ 756579324aSTerje Bergstrom pb->fence = pb->size_bytes - 8; 766579324aSTerje Bergstrom pb->pos = 0; 776579324aSTerje Bergstrom 786579324aSTerje Bergstrom /* allocate and map pushbuffer memory */ 796579324aSTerje Bergstrom pb->mapped = dma_alloc_writecombine(host1x->dev, pb->size_bytes + 4, 806579324aSTerje Bergstrom &pb->phys, GFP_KERNEL); 816579324aSTerje Bergstrom if (!pb->mapped) 826579324aSTerje Bergstrom goto fail; 836579324aSTerje Bergstrom 846579324aSTerje Bergstrom host1x_hw_pushbuffer_init(host1x, pb); 856579324aSTerje Bergstrom 866579324aSTerje Bergstrom return 0; 876579324aSTerje Bergstrom 886579324aSTerje Bergstrom fail: 896579324aSTerje Bergstrom host1x_pushbuffer_destroy(pb); 906579324aSTerje Bergstrom return -ENOMEM; 916579324aSTerje Bergstrom } 926579324aSTerje Bergstrom 936579324aSTerje Bergstrom /* 946579324aSTerje Bergstrom * Push two words to the push buffer 956579324aSTerje Bergstrom * Caller must ensure push buffer is not full 966579324aSTerje Bergstrom */ 976579324aSTerje Bergstrom static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2) 986579324aSTerje Bergstrom { 996579324aSTerje Bergstrom u32 pos = pb->pos; 1006579324aSTerje Bergstrom u32 *p = (u32 *)((u32)pb->mapped + pos); 1016579324aSTerje Bergstrom WARN_ON(pos == pb->fence); 1026579324aSTerje Bergstrom *(p++) = op1; 1036579324aSTerje Bergstrom *(p++) = op2; 1046579324aSTerje Bergstrom pb->pos = (pos + 8) & (pb->size_bytes - 1); 1056579324aSTerje Bergstrom } 1066579324aSTerje Bergstrom 1076579324aSTerje Bergstrom /* 1086579324aSTerje Bergstrom * Pop a number of two word slots from the push buffer 1096579324aSTerje Bergstrom * Caller must ensure push buffer is not empty 1106579324aSTerje Bergstrom */ 1116579324aSTerje Bergstrom static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots) 1126579324aSTerje Bergstrom { 1136579324aSTerje Bergstrom /* Advance the next write position */ 1146579324aSTerje Bergstrom pb->fence = (pb->fence + slots * 8) & (pb->size_bytes - 1); 1156579324aSTerje Bergstrom } 1166579324aSTerje Bergstrom 1176579324aSTerje Bergstrom /* 1186579324aSTerje Bergstrom * Return the number of two word slots free in the push buffer 1196579324aSTerje Bergstrom */ 1206579324aSTerje Bergstrom static u32 host1x_pushbuffer_space(struct push_buffer *pb) 1216579324aSTerje Bergstrom { 1226579324aSTerje Bergstrom return ((pb->fence - pb->pos) & (pb->size_bytes - 1)) / 8; 1236579324aSTerje Bergstrom } 1246579324aSTerje Bergstrom 1256579324aSTerje Bergstrom /* 1266579324aSTerje Bergstrom * Sleep (if necessary) until the requested event happens 1276579324aSTerje Bergstrom * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. 1286579324aSTerje Bergstrom * - Returns 1 1296579324aSTerje Bergstrom * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer 1306579324aSTerje Bergstrom * - Return the amount of space (> 0) 1316579324aSTerje Bergstrom * Must be called with the cdma lock held. 1326579324aSTerje Bergstrom */ 1336579324aSTerje Bergstrom unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, 1346579324aSTerje Bergstrom enum cdma_event event) 1356579324aSTerje Bergstrom { 1366579324aSTerje Bergstrom for (;;) { 1376579324aSTerje Bergstrom unsigned int space; 1386579324aSTerje Bergstrom 1396579324aSTerje Bergstrom if (event == CDMA_EVENT_SYNC_QUEUE_EMPTY) 1406579324aSTerje Bergstrom space = list_empty(&cdma->sync_queue) ? 1 : 0; 1416579324aSTerje Bergstrom else if (event == CDMA_EVENT_PUSH_BUFFER_SPACE) { 1426579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 1436579324aSTerje Bergstrom space = host1x_pushbuffer_space(pb); 1446579324aSTerje Bergstrom } else { 1456579324aSTerje Bergstrom WARN_ON(1); 1466579324aSTerje Bergstrom return -EINVAL; 1476579324aSTerje Bergstrom } 1486579324aSTerje Bergstrom 1496579324aSTerje Bergstrom if (space) 1506579324aSTerje Bergstrom return space; 1516579324aSTerje Bergstrom 1526579324aSTerje Bergstrom trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), 1536579324aSTerje Bergstrom event); 1546579324aSTerje Bergstrom 1556579324aSTerje Bergstrom /* If somebody has managed to already start waiting, yield */ 1566579324aSTerje Bergstrom if (cdma->event != CDMA_EVENT_NONE) { 1576579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 1586579324aSTerje Bergstrom schedule(); 1596579324aSTerje Bergstrom mutex_lock(&cdma->lock); 1606579324aSTerje Bergstrom continue; 1616579324aSTerje Bergstrom } 1626579324aSTerje Bergstrom cdma->event = event; 1636579324aSTerje Bergstrom 1646579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 1656579324aSTerje Bergstrom down(&cdma->sem); 1666579324aSTerje Bergstrom mutex_lock(&cdma->lock); 1676579324aSTerje Bergstrom } 1686579324aSTerje Bergstrom return 0; 1696579324aSTerje Bergstrom } 1706579324aSTerje Bergstrom 1716579324aSTerje Bergstrom /* 1726579324aSTerje Bergstrom * Start timer that tracks the time spent by the job. 1736579324aSTerje Bergstrom * Must be called with the cdma lock held. 1746579324aSTerje Bergstrom */ 1756579324aSTerje Bergstrom static void cdma_start_timer_locked(struct host1x_cdma *cdma, 1766579324aSTerje Bergstrom struct host1x_job *job) 1776579324aSTerje Bergstrom { 1786579324aSTerje Bergstrom struct host1x *host = cdma_to_host1x(cdma); 1796579324aSTerje Bergstrom 1806579324aSTerje Bergstrom if (cdma->timeout.client) { 1816579324aSTerje Bergstrom /* timer already started */ 1826579324aSTerje Bergstrom return; 1836579324aSTerje Bergstrom } 1846579324aSTerje Bergstrom 1856579324aSTerje Bergstrom cdma->timeout.client = job->client; 1866579324aSTerje Bergstrom cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id); 1876579324aSTerje Bergstrom cdma->timeout.syncpt_val = job->syncpt_end; 1886579324aSTerje Bergstrom cdma->timeout.start_ktime = ktime_get(); 1896579324aSTerje Bergstrom 1906579324aSTerje Bergstrom schedule_delayed_work(&cdma->timeout.wq, 1916579324aSTerje Bergstrom msecs_to_jiffies(job->timeout)); 1926579324aSTerje Bergstrom } 1936579324aSTerje Bergstrom 1946579324aSTerje Bergstrom /* 1956579324aSTerje Bergstrom * Stop timer when a buffer submission completes. 1966579324aSTerje Bergstrom * Must be called with the cdma lock held. 1976579324aSTerje Bergstrom */ 1986579324aSTerje Bergstrom static void stop_cdma_timer_locked(struct host1x_cdma *cdma) 1996579324aSTerje Bergstrom { 2006579324aSTerje Bergstrom cancel_delayed_work(&cdma->timeout.wq); 2016579324aSTerje Bergstrom cdma->timeout.client = 0; 2026579324aSTerje Bergstrom } 2036579324aSTerje Bergstrom 2046579324aSTerje Bergstrom /* 2056579324aSTerje Bergstrom * For all sync queue entries that have already finished according to the 2066579324aSTerje Bergstrom * current sync point registers: 2076579324aSTerje Bergstrom * - unpin & unref their mems 2086579324aSTerje Bergstrom * - pop their push buffer slots 2096579324aSTerje Bergstrom * - remove them from the sync queue 2106579324aSTerje Bergstrom * This is normally called from the host code's worker thread, but can be 2116579324aSTerje Bergstrom * called manually if necessary. 2126579324aSTerje Bergstrom * Must be called with the cdma lock held. 2136579324aSTerje Bergstrom */ 2146579324aSTerje Bergstrom static void update_cdma_locked(struct host1x_cdma *cdma) 2156579324aSTerje Bergstrom { 2166579324aSTerje Bergstrom bool signal = false; 2176579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 2186579324aSTerje Bergstrom struct host1x_job *job, *n; 2196579324aSTerje Bergstrom 2206579324aSTerje Bergstrom /* If CDMA is stopped, queue is cleared and we can return */ 2216579324aSTerje Bergstrom if (!cdma->running) 2226579324aSTerje Bergstrom return; 2236579324aSTerje Bergstrom 2246579324aSTerje Bergstrom /* 2256579324aSTerje Bergstrom * Walk the sync queue, reading the sync point registers as necessary, 2266579324aSTerje Bergstrom * to consume as many sync queue entries as possible without blocking 2276579324aSTerje Bergstrom */ 2286579324aSTerje Bergstrom list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { 2296579324aSTerje Bergstrom struct host1x_syncpt *sp = 2306579324aSTerje Bergstrom host1x_syncpt_get(host1x, job->syncpt_id); 2316579324aSTerje Bergstrom 2326579324aSTerje Bergstrom /* Check whether this syncpt has completed, and bail if not */ 2336579324aSTerje Bergstrom if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) { 2346579324aSTerje Bergstrom /* Start timer on next pending syncpt */ 2356579324aSTerje Bergstrom if (job->timeout) 2366579324aSTerje Bergstrom cdma_start_timer_locked(cdma, job); 2376579324aSTerje Bergstrom break; 2386579324aSTerje Bergstrom } 2396579324aSTerje Bergstrom 2406579324aSTerje Bergstrom /* Cancel timeout, when a buffer completes */ 2416579324aSTerje Bergstrom if (cdma->timeout.client) 2426579324aSTerje Bergstrom stop_cdma_timer_locked(cdma); 2436579324aSTerje Bergstrom 2446579324aSTerje Bergstrom /* Unpin the memory */ 2456579324aSTerje Bergstrom host1x_job_unpin(job); 2466579324aSTerje Bergstrom 2476579324aSTerje Bergstrom /* Pop push buffer slots */ 2486579324aSTerje Bergstrom if (job->num_slots) { 2496579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 2506579324aSTerje Bergstrom host1x_pushbuffer_pop(pb, job->num_slots); 2516579324aSTerje Bergstrom if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) 2526579324aSTerje Bergstrom signal = true; 2536579324aSTerje Bergstrom } 2546579324aSTerje Bergstrom 2556579324aSTerje Bergstrom list_del(&job->list); 2566579324aSTerje Bergstrom host1x_job_put(job); 2576579324aSTerje Bergstrom } 2586579324aSTerje Bergstrom 2596579324aSTerje Bergstrom if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && 2606579324aSTerje Bergstrom list_empty(&cdma->sync_queue)) 2616579324aSTerje Bergstrom signal = true; 2626579324aSTerje Bergstrom 2636579324aSTerje Bergstrom if (signal) { 2646579324aSTerje Bergstrom cdma->event = CDMA_EVENT_NONE; 2656579324aSTerje Bergstrom up(&cdma->sem); 2666579324aSTerje Bergstrom } 2676579324aSTerje Bergstrom } 2686579324aSTerje Bergstrom 2696579324aSTerje Bergstrom void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, 2706579324aSTerje Bergstrom struct device *dev) 2716579324aSTerje Bergstrom { 2726579324aSTerje Bergstrom u32 restart_addr; 2736579324aSTerje Bergstrom u32 syncpt_incrs; 2746579324aSTerje Bergstrom struct host1x_job *job = NULL; 2756579324aSTerje Bergstrom u32 syncpt_val; 2766579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 2776579324aSTerje Bergstrom 2786579324aSTerje Bergstrom syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 2796579324aSTerje Bergstrom 2806579324aSTerje Bergstrom dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", 2816579324aSTerje Bergstrom __func__, syncpt_val); 2826579324aSTerje Bergstrom 2836579324aSTerje Bergstrom /* 2846579324aSTerje Bergstrom * Move the sync_queue read pointer to the first entry that hasn't 2856579324aSTerje Bergstrom * completed based on the current HW syncpt value. It's likely there 2866579324aSTerje Bergstrom * won't be any (i.e. we're still at the head), but covers the case 2876579324aSTerje Bergstrom * where a syncpt incr happens just prior/during the teardown. 2886579324aSTerje Bergstrom */ 2896579324aSTerje Bergstrom 2906579324aSTerje Bergstrom dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", 2916579324aSTerje Bergstrom __func__); 2926579324aSTerje Bergstrom 2936579324aSTerje Bergstrom list_for_each_entry(job, &cdma->sync_queue, list) { 2946579324aSTerje Bergstrom if (syncpt_val < job->syncpt_end) 2956579324aSTerje Bergstrom break; 2966579324aSTerje Bergstrom 2976579324aSTerje Bergstrom host1x_job_dump(dev, job); 2986579324aSTerje Bergstrom } 2996579324aSTerje Bergstrom 3006579324aSTerje Bergstrom /* 3016579324aSTerje Bergstrom * Walk the sync_queue, first incrementing with the CPU syncpts that 3026579324aSTerje Bergstrom * are partially executed (the first buffer) or fully skipped while 3036579324aSTerje Bergstrom * still in the current context (slots are also NOP-ed). 3046579324aSTerje Bergstrom * 3056579324aSTerje Bergstrom * At the point contexts are interleaved, syncpt increments must be 3066579324aSTerje Bergstrom * done inline with the pushbuffer from a GATHER buffer to maintain 3076579324aSTerje Bergstrom * the order (slots are modified to be a GATHER of syncpt incrs). 3086579324aSTerje Bergstrom * 3096579324aSTerje Bergstrom * Note: save in restart_addr the location where the timed out buffer 3106579324aSTerje Bergstrom * started in the PB, so we can start the refetch from there (with the 3116579324aSTerje Bergstrom * modified NOP-ed PB slots). This lets things appear to have completed 3126579324aSTerje Bergstrom * properly for this buffer and resources are freed. 3136579324aSTerje Bergstrom */ 3146579324aSTerje Bergstrom 3156579324aSTerje Bergstrom dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n", 3166579324aSTerje Bergstrom __func__); 3176579324aSTerje Bergstrom 3186579324aSTerje Bergstrom if (!list_empty(&cdma->sync_queue)) 3196579324aSTerje Bergstrom restart_addr = job->first_get; 3206579324aSTerje Bergstrom else 3216579324aSTerje Bergstrom restart_addr = cdma->last_pos; 3226579324aSTerje Bergstrom 3236579324aSTerje Bergstrom /* do CPU increments as long as this context continues */ 3246579324aSTerje Bergstrom list_for_each_entry_from(job, &cdma->sync_queue, list) { 3256579324aSTerje Bergstrom /* different context, gets us out of this loop */ 3266579324aSTerje Bergstrom if (job->client != cdma->timeout.client) 3276579324aSTerje Bergstrom break; 3286579324aSTerje Bergstrom 3296579324aSTerje Bergstrom /* won't need a timeout when replayed */ 3306579324aSTerje Bergstrom job->timeout = 0; 3316579324aSTerje Bergstrom 3326579324aSTerje Bergstrom syncpt_incrs = job->syncpt_end - syncpt_val; 3336579324aSTerje Bergstrom dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); 3346579324aSTerje Bergstrom 3356579324aSTerje Bergstrom host1x_job_dump(dev, job); 3366579324aSTerje Bergstrom 3376579324aSTerje Bergstrom /* safe to use CPU to incr syncpts */ 3386579324aSTerje Bergstrom host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, 3396579324aSTerje Bergstrom syncpt_incrs, job->syncpt_end, 3406579324aSTerje Bergstrom job->num_slots); 3416579324aSTerje Bergstrom 3426579324aSTerje Bergstrom syncpt_val += syncpt_incrs; 3436579324aSTerje Bergstrom } 3446579324aSTerje Bergstrom 3456579324aSTerje Bergstrom /* The following sumbits from the same client may be dependent on the 3466579324aSTerje Bergstrom * failed submit and therefore they may fail. Force a small timeout 3476579324aSTerje Bergstrom * to make the queue cleanup faster */ 3486579324aSTerje Bergstrom 3496579324aSTerje Bergstrom list_for_each_entry_from(job, &cdma->sync_queue, list) 3506579324aSTerje Bergstrom if (job->client == cdma->timeout.client) 3516579324aSTerje Bergstrom job->timeout = min_t(unsigned int, job->timeout, 500); 3526579324aSTerje Bergstrom 3536579324aSTerje Bergstrom dev_dbg(dev, "%s: finished sync_queue modification\n", __func__); 3546579324aSTerje Bergstrom 3556579324aSTerje Bergstrom /* roll back DMAGET and start up channel again */ 3566579324aSTerje Bergstrom host1x_hw_cdma_resume(host1x, cdma, restart_addr); 3576579324aSTerje Bergstrom } 3586579324aSTerje Bergstrom 3596579324aSTerje Bergstrom /* 3606579324aSTerje Bergstrom * Create a cdma 3616579324aSTerje Bergstrom */ 3626579324aSTerje Bergstrom int host1x_cdma_init(struct host1x_cdma *cdma) 3636579324aSTerje Bergstrom { 3646579324aSTerje Bergstrom int err; 3656579324aSTerje Bergstrom 3666579324aSTerje Bergstrom mutex_init(&cdma->lock); 3676579324aSTerje Bergstrom sema_init(&cdma->sem, 0); 3686579324aSTerje Bergstrom 3696579324aSTerje Bergstrom INIT_LIST_HEAD(&cdma->sync_queue); 3706579324aSTerje Bergstrom 3716579324aSTerje Bergstrom cdma->event = CDMA_EVENT_NONE; 3726579324aSTerje Bergstrom cdma->running = false; 3736579324aSTerje Bergstrom cdma->torndown = false; 3746579324aSTerje Bergstrom 3756579324aSTerje Bergstrom err = host1x_pushbuffer_init(&cdma->push_buffer); 3766579324aSTerje Bergstrom if (err) 3776579324aSTerje Bergstrom return err; 3786579324aSTerje Bergstrom return 0; 3796579324aSTerje Bergstrom } 3806579324aSTerje Bergstrom 3816579324aSTerje Bergstrom /* 3826579324aSTerje Bergstrom * Destroy a cdma 3836579324aSTerje Bergstrom */ 3846579324aSTerje Bergstrom int host1x_cdma_deinit(struct host1x_cdma *cdma) 3856579324aSTerje Bergstrom { 3866579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 3876579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 3886579324aSTerje Bergstrom 3896579324aSTerje Bergstrom if (cdma->running) { 3906579324aSTerje Bergstrom pr_warn("%s: CDMA still running\n", __func__); 3916579324aSTerje Bergstrom return -EBUSY; 3926579324aSTerje Bergstrom } 3936579324aSTerje Bergstrom 3946579324aSTerje Bergstrom host1x_pushbuffer_destroy(pb); 3956579324aSTerje Bergstrom host1x_hw_cdma_timeout_destroy(host1x, cdma); 3966579324aSTerje Bergstrom 3976579324aSTerje Bergstrom return 0; 3986579324aSTerje Bergstrom } 3996579324aSTerje Bergstrom 4006579324aSTerje Bergstrom /* 4016579324aSTerje Bergstrom * Begin a cdma submit 4026579324aSTerje Bergstrom */ 4036579324aSTerje Bergstrom int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job) 4046579324aSTerje Bergstrom { 4056579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 4066579324aSTerje Bergstrom 4076579324aSTerje Bergstrom mutex_lock(&cdma->lock); 4086579324aSTerje Bergstrom 4096579324aSTerje Bergstrom if (job->timeout) { 4106579324aSTerje Bergstrom /* init state on first submit with timeout value */ 4116579324aSTerje Bergstrom if (!cdma->timeout.initialized) { 4126579324aSTerje Bergstrom int err; 4136579324aSTerje Bergstrom err = host1x_hw_cdma_timeout_init(host1x, cdma, 4146579324aSTerje Bergstrom job->syncpt_id); 4156579324aSTerje Bergstrom if (err) { 4166579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 4176579324aSTerje Bergstrom return err; 4186579324aSTerje Bergstrom } 4196579324aSTerje Bergstrom } 4206579324aSTerje Bergstrom } 4216579324aSTerje Bergstrom if (!cdma->running) 4226579324aSTerje Bergstrom host1x_hw_cdma_start(host1x, cdma); 4236579324aSTerje Bergstrom 4246579324aSTerje Bergstrom cdma->slots_free = 0; 4256579324aSTerje Bergstrom cdma->slots_used = 0; 4266579324aSTerje Bergstrom cdma->first_get = cdma->push_buffer.pos; 4276579324aSTerje Bergstrom 4286579324aSTerje Bergstrom trace_host1x_cdma_begin(dev_name(job->channel->dev)); 4296579324aSTerje Bergstrom return 0; 4306579324aSTerje Bergstrom } 4316579324aSTerje Bergstrom 4326579324aSTerje Bergstrom /* 4336579324aSTerje Bergstrom * Push two words into a push buffer slot 4346579324aSTerje Bergstrom * Blocks as necessary if the push buffer is full. 4356579324aSTerje Bergstrom */ 4366579324aSTerje Bergstrom void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2) 4376579324aSTerje Bergstrom { 4386579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 4396579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 4406579324aSTerje Bergstrom u32 slots_free = cdma->slots_free; 4416579324aSTerje Bergstrom 4426236451dSTerje Bergstrom if (host1x_debug_trace_cmdbuf) 4436236451dSTerje Bergstrom trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev), 4446236451dSTerje Bergstrom op1, op2); 4456236451dSTerje Bergstrom 4466579324aSTerje Bergstrom if (slots_free == 0) { 4476579324aSTerje Bergstrom host1x_hw_cdma_flush(host1x, cdma); 4486579324aSTerje Bergstrom slots_free = host1x_cdma_wait_locked(cdma, 4496579324aSTerje Bergstrom CDMA_EVENT_PUSH_BUFFER_SPACE); 4506579324aSTerje Bergstrom } 4516579324aSTerje Bergstrom cdma->slots_free = slots_free - 1; 4526579324aSTerje Bergstrom cdma->slots_used++; 4536579324aSTerje Bergstrom host1x_pushbuffer_push(pb, op1, op2); 4546579324aSTerje Bergstrom } 4556579324aSTerje Bergstrom 4566579324aSTerje Bergstrom /* 4576579324aSTerje Bergstrom * End a cdma submit 4586579324aSTerje Bergstrom * Kick off DMA, add job to the sync queue, and a number of slots to be freed 4596579324aSTerje Bergstrom * from the pushbuffer. The handles for a submit must all be pinned at the same 4606579324aSTerje Bergstrom * time, but they can be unpinned in smaller chunks. 4616579324aSTerje Bergstrom */ 4626579324aSTerje Bergstrom void host1x_cdma_end(struct host1x_cdma *cdma, 4636579324aSTerje Bergstrom struct host1x_job *job) 4646579324aSTerje Bergstrom { 4656579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 4666579324aSTerje Bergstrom bool idle = list_empty(&cdma->sync_queue); 4676579324aSTerje Bergstrom 4686579324aSTerje Bergstrom host1x_hw_cdma_flush(host1x, cdma); 4696579324aSTerje Bergstrom 4706579324aSTerje Bergstrom job->first_get = cdma->first_get; 4716579324aSTerje Bergstrom job->num_slots = cdma->slots_used; 4726579324aSTerje Bergstrom host1x_job_get(job); 4736579324aSTerje Bergstrom list_add_tail(&job->list, &cdma->sync_queue); 4746579324aSTerje Bergstrom 4756579324aSTerje Bergstrom /* start timer on idle -> active transitions */ 4766579324aSTerje Bergstrom if (job->timeout && idle) 4776579324aSTerje Bergstrom cdma_start_timer_locked(cdma, job); 4786579324aSTerje Bergstrom 4796579324aSTerje Bergstrom trace_host1x_cdma_end(dev_name(job->channel->dev)); 4806579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 4816579324aSTerje Bergstrom } 4826579324aSTerje Bergstrom 4836579324aSTerje Bergstrom /* 4846579324aSTerje Bergstrom * Update cdma state according to current sync point values 4856579324aSTerje Bergstrom */ 4866579324aSTerje Bergstrom void host1x_cdma_update(struct host1x_cdma *cdma) 4876579324aSTerje Bergstrom { 4886579324aSTerje Bergstrom mutex_lock(&cdma->lock); 4896579324aSTerje Bergstrom update_cdma_locked(cdma); 4906579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 4916579324aSTerje Bergstrom } 492