16579324aSTerje Bergstrom /* 26579324aSTerje Bergstrom * Tegra host1x Command DMA 36579324aSTerje Bergstrom * 46579324aSTerje Bergstrom * Copyright (c) 2010-2013, NVIDIA Corporation. 56579324aSTerje Bergstrom * 66579324aSTerje Bergstrom * This program is free software; you can redistribute it and/or modify it 76579324aSTerje Bergstrom * under the terms and conditions of the GNU General Public License, 86579324aSTerje Bergstrom * version 2, as published by the Free Software Foundation. 96579324aSTerje Bergstrom * 106579324aSTerje Bergstrom * This program is distributed in the hope it will be useful, but WITHOUT 116579324aSTerje Bergstrom * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 126579324aSTerje Bergstrom * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 136579324aSTerje Bergstrom * more details. 146579324aSTerje Bergstrom * 156579324aSTerje Bergstrom * You should have received a copy of the GNU General Public License 166579324aSTerje Bergstrom * along with this program. If not, see <http://www.gnu.org/licenses/>. 176579324aSTerje Bergstrom */ 186579324aSTerje Bergstrom 196579324aSTerje Bergstrom 206579324aSTerje Bergstrom #include <asm/cacheflush.h> 216579324aSTerje Bergstrom #include <linux/device.h> 226579324aSTerje Bergstrom #include <linux/dma-mapping.h> 2335d747a8SThierry Reding #include <linux/host1x.h> 246579324aSTerje Bergstrom #include <linux/interrupt.h> 256579324aSTerje Bergstrom #include <linux/kernel.h> 266579324aSTerje Bergstrom #include <linux/kfifo.h> 276579324aSTerje Bergstrom #include <linux/slab.h> 286579324aSTerje Bergstrom #include <trace/events/host1x.h> 296579324aSTerje Bergstrom 306579324aSTerje Bergstrom #include "cdma.h" 316579324aSTerje Bergstrom #include "channel.h" 326579324aSTerje Bergstrom #include "dev.h" 336579324aSTerje Bergstrom #include "debug.h" 346579324aSTerje Bergstrom #include "job.h" 356579324aSTerje Bergstrom 366579324aSTerje Bergstrom /* 376579324aSTerje Bergstrom * push_buffer 386579324aSTerje Bergstrom * 396579324aSTerje Bergstrom * The push buffer is a circular array of words to be fetched by command DMA. 406579324aSTerje Bergstrom * Note that it works slightly differently to the sync queue; fence == pos 416579324aSTerje Bergstrom * means that the push buffer is full, not empty. 426579324aSTerje Bergstrom */ 436579324aSTerje Bergstrom 446579324aSTerje Bergstrom #define HOST1X_PUSHBUFFER_SLOTS 512 456579324aSTerje Bergstrom 466579324aSTerje Bergstrom /* 476579324aSTerje Bergstrom * Clean up push buffer resources 486579324aSTerje Bergstrom */ 496579324aSTerje Bergstrom static void host1x_pushbuffer_destroy(struct push_buffer *pb) 506579324aSTerje Bergstrom { 516579324aSTerje Bergstrom struct host1x_cdma *cdma = pb_to_cdma(pb); 526579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 536579324aSTerje Bergstrom 54404bfb78SMikko Perttunen if (!pb->phys) 55404bfb78SMikko Perttunen return; 56404bfb78SMikko Perttunen 57404bfb78SMikko Perttunen if (host1x->domain) { 58404bfb78SMikko Perttunen iommu_unmap(host1x->domain, pb->dma, pb->alloc_size); 59404bfb78SMikko Perttunen free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma)); 60404bfb78SMikko Perttunen } 61404bfb78SMikko Perttunen 62404bfb78SMikko Perttunen dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys); 636579324aSTerje Bergstrom 646579324aSTerje Bergstrom pb->mapped = NULL; 656579324aSTerje Bergstrom pb->phys = 0; 666579324aSTerje Bergstrom } 676579324aSTerje Bergstrom 686579324aSTerje Bergstrom /* 696579324aSTerje Bergstrom * Init push buffer resources 706579324aSTerje Bergstrom */ 716579324aSTerje Bergstrom static int host1x_pushbuffer_init(struct push_buffer *pb) 726579324aSTerje Bergstrom { 736579324aSTerje Bergstrom struct host1x_cdma *cdma = pb_to_cdma(pb); 746579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 75404bfb78SMikko Perttunen struct iova *alloc; 76404bfb78SMikko Perttunen u32 size; 77404bfb78SMikko Perttunen int err; 786579324aSTerje Bergstrom 796579324aSTerje Bergstrom pb->mapped = NULL; 806579324aSTerje Bergstrom pb->phys = 0; 81404bfb78SMikko Perttunen pb->size = HOST1X_PUSHBUFFER_SLOTS * 8; 82404bfb78SMikko Perttunen 83404bfb78SMikko Perttunen size = pb->size + 4; 846579324aSTerje Bergstrom 856579324aSTerje Bergstrom /* initialize buffer pointers */ 86404bfb78SMikko Perttunen pb->fence = pb->size - 8; 876579324aSTerje Bergstrom pb->pos = 0; 886579324aSTerje Bergstrom 89404bfb78SMikko Perttunen if (host1x->domain) { 90404bfb78SMikko Perttunen unsigned long shift; 91404bfb78SMikko Perttunen 92404bfb78SMikko Perttunen size = iova_align(&host1x->iova, size); 93404bfb78SMikko Perttunen 94404bfb78SMikko Perttunen pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, 95f6e45661SLuis R. Rodriguez GFP_KERNEL); 966579324aSTerje Bergstrom if (!pb->mapped) 97404bfb78SMikko Perttunen return -ENOMEM; 98404bfb78SMikko Perttunen 99404bfb78SMikko Perttunen shift = iova_shift(&host1x->iova); 100404bfb78SMikko Perttunen alloc = alloc_iova(&host1x->iova, size >> shift, 101404bfb78SMikko Perttunen host1x->iova_end >> shift, true); 102404bfb78SMikko Perttunen if (!alloc) { 103404bfb78SMikko Perttunen err = -ENOMEM; 104404bfb78SMikko Perttunen goto iommu_free_mem; 105404bfb78SMikko Perttunen } 106404bfb78SMikko Perttunen 107404bfb78SMikko Perttunen pb->dma = iova_dma_addr(&host1x->iova, alloc); 108404bfb78SMikko Perttunen err = iommu_map(host1x->domain, pb->dma, pb->phys, size, 109404bfb78SMikko Perttunen IOMMU_READ); 110404bfb78SMikko Perttunen if (err) 111404bfb78SMikko Perttunen goto iommu_free_iova; 112404bfb78SMikko Perttunen } else { 113404bfb78SMikko Perttunen pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, 114404bfb78SMikko Perttunen GFP_KERNEL); 115404bfb78SMikko Perttunen if (!pb->mapped) 116404bfb78SMikko Perttunen return -ENOMEM; 117404bfb78SMikko Perttunen 118404bfb78SMikko Perttunen pb->dma = pb->phys; 119404bfb78SMikko Perttunen } 120404bfb78SMikko Perttunen 121404bfb78SMikko Perttunen pb->alloc_size = size; 1226579324aSTerje Bergstrom 1236579324aSTerje Bergstrom host1x_hw_pushbuffer_init(host1x, pb); 1246579324aSTerje Bergstrom 1256579324aSTerje Bergstrom return 0; 1266579324aSTerje Bergstrom 127404bfb78SMikko Perttunen iommu_free_iova: 128404bfb78SMikko Perttunen __free_iova(&host1x->iova, alloc); 129404bfb78SMikko Perttunen iommu_free_mem: 13027db6a00SDmitry Osipenko dma_free_wc(host1x->dev, size, pb->mapped, pb->phys); 131404bfb78SMikko Perttunen 132404bfb78SMikko Perttunen return err; 1336579324aSTerje Bergstrom } 1346579324aSTerje Bergstrom 1356579324aSTerje Bergstrom /* 1366579324aSTerje Bergstrom * Push two words to the push buffer 1376579324aSTerje Bergstrom * Caller must ensure push buffer is not full 1386579324aSTerje Bergstrom */ 1396579324aSTerje Bergstrom static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2) 1406579324aSTerje Bergstrom { 141ebb2475cSThierry Reding u32 *p = (u32 *)((void *)pb->mapped + pb->pos); 142ebb2475cSThierry Reding 143ebb2475cSThierry Reding WARN_ON(pb->pos == pb->fence); 1446579324aSTerje Bergstrom *(p++) = op1; 1456579324aSTerje Bergstrom *(p++) = op2; 146404bfb78SMikko Perttunen pb->pos = (pb->pos + 8) & (pb->size - 1); 1476579324aSTerje Bergstrom } 1486579324aSTerje Bergstrom 1496579324aSTerje Bergstrom /* 1506579324aSTerje Bergstrom * Pop a number of two word slots from the push buffer 1516579324aSTerje Bergstrom * Caller must ensure push buffer is not empty 1526579324aSTerje Bergstrom */ 1536579324aSTerje Bergstrom static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots) 1546579324aSTerje Bergstrom { 1556579324aSTerje Bergstrom /* Advance the next write position */ 156404bfb78SMikko Perttunen pb->fence = (pb->fence + slots * 8) & (pb->size - 1); 1576579324aSTerje Bergstrom } 1586579324aSTerje Bergstrom 1596579324aSTerje Bergstrom /* 1606579324aSTerje Bergstrom * Return the number of two word slots free in the push buffer 1616579324aSTerje Bergstrom */ 1626579324aSTerje Bergstrom static u32 host1x_pushbuffer_space(struct push_buffer *pb) 1636579324aSTerje Bergstrom { 164404bfb78SMikko Perttunen return ((pb->fence - pb->pos) & (pb->size - 1)) / 8; 1656579324aSTerje Bergstrom } 1666579324aSTerje Bergstrom 1676579324aSTerje Bergstrom /* 1686579324aSTerje Bergstrom * Sleep (if necessary) until the requested event happens 1696579324aSTerje Bergstrom * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. 1706579324aSTerje Bergstrom * - Returns 1 1716579324aSTerje Bergstrom * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer 1726579324aSTerje Bergstrom * - Return the amount of space (> 0) 1736579324aSTerje Bergstrom * Must be called with the cdma lock held. 1746579324aSTerje Bergstrom */ 1756579324aSTerje Bergstrom unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, 1766579324aSTerje Bergstrom enum cdma_event event) 1776579324aSTerje Bergstrom { 1786579324aSTerje Bergstrom for (;;) { 1790b8070d1SThierry Reding struct push_buffer *pb = &cdma->push_buffer; 1806579324aSTerje Bergstrom unsigned int space; 1816579324aSTerje Bergstrom 1820b8070d1SThierry Reding switch (event) { 1830b8070d1SThierry Reding case CDMA_EVENT_SYNC_QUEUE_EMPTY: 1846579324aSTerje Bergstrom space = list_empty(&cdma->sync_queue) ? 1 : 0; 1850b8070d1SThierry Reding break; 1860b8070d1SThierry Reding 1870b8070d1SThierry Reding case CDMA_EVENT_PUSH_BUFFER_SPACE: 1886579324aSTerje Bergstrom space = host1x_pushbuffer_space(pb); 1890b8070d1SThierry Reding break; 1900b8070d1SThierry Reding 1910b8070d1SThierry Reding default: 1926579324aSTerje Bergstrom WARN_ON(1); 1936579324aSTerje Bergstrom return -EINVAL; 1946579324aSTerje Bergstrom } 1956579324aSTerje Bergstrom 1966579324aSTerje Bergstrom if (space) 1976579324aSTerje Bergstrom return space; 1986579324aSTerje Bergstrom 1996579324aSTerje Bergstrom trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), 2006579324aSTerje Bergstrom event); 2016579324aSTerje Bergstrom 2026579324aSTerje Bergstrom /* If somebody has managed to already start waiting, yield */ 2036579324aSTerje Bergstrom if (cdma->event != CDMA_EVENT_NONE) { 2046579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 2056579324aSTerje Bergstrom schedule(); 2066579324aSTerje Bergstrom mutex_lock(&cdma->lock); 2076579324aSTerje Bergstrom continue; 2086579324aSTerje Bergstrom } 2090b8070d1SThierry Reding 2106579324aSTerje Bergstrom cdma->event = event; 2116579324aSTerje Bergstrom 2126579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 2136579324aSTerje Bergstrom down(&cdma->sem); 2146579324aSTerje Bergstrom mutex_lock(&cdma->lock); 2156579324aSTerje Bergstrom } 2160b8070d1SThierry Reding 2176579324aSTerje Bergstrom return 0; 2186579324aSTerje Bergstrom } 2196579324aSTerje Bergstrom 2206579324aSTerje Bergstrom /* 2216579324aSTerje Bergstrom * Start timer that tracks the time spent by the job. 2226579324aSTerje Bergstrom * Must be called with the cdma lock held. 2236579324aSTerje Bergstrom */ 2246579324aSTerje Bergstrom static void cdma_start_timer_locked(struct host1x_cdma *cdma, 2256579324aSTerje Bergstrom struct host1x_job *job) 2266579324aSTerje Bergstrom { 2276579324aSTerje Bergstrom struct host1x *host = cdma_to_host1x(cdma); 2286579324aSTerje Bergstrom 2296579324aSTerje Bergstrom if (cdma->timeout.client) { 2306579324aSTerje Bergstrom /* timer already started */ 2316579324aSTerje Bergstrom return; 2326579324aSTerje Bergstrom } 2336579324aSTerje Bergstrom 2346579324aSTerje Bergstrom cdma->timeout.client = job->client; 2356579324aSTerje Bergstrom cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id); 2366579324aSTerje Bergstrom cdma->timeout.syncpt_val = job->syncpt_end; 2376579324aSTerje Bergstrom cdma->timeout.start_ktime = ktime_get(); 2386579324aSTerje Bergstrom 2396579324aSTerje Bergstrom schedule_delayed_work(&cdma->timeout.wq, 2406579324aSTerje Bergstrom msecs_to_jiffies(job->timeout)); 2416579324aSTerje Bergstrom } 2426579324aSTerje Bergstrom 2436579324aSTerje Bergstrom /* 2446579324aSTerje Bergstrom * Stop timer when a buffer submission completes. 2456579324aSTerje Bergstrom * Must be called with the cdma lock held. 2466579324aSTerje Bergstrom */ 2476579324aSTerje Bergstrom static void stop_cdma_timer_locked(struct host1x_cdma *cdma) 2486579324aSTerje Bergstrom { 2496579324aSTerje Bergstrom cancel_delayed_work(&cdma->timeout.wq); 2506579324aSTerje Bergstrom cdma->timeout.client = 0; 2516579324aSTerje Bergstrom } 2526579324aSTerje Bergstrom 2536579324aSTerje Bergstrom /* 2546579324aSTerje Bergstrom * For all sync queue entries that have already finished according to the 2556579324aSTerje Bergstrom * current sync point registers: 2566579324aSTerje Bergstrom * - unpin & unref their mems 2576579324aSTerje Bergstrom * - pop their push buffer slots 2586579324aSTerje Bergstrom * - remove them from the sync queue 2596579324aSTerje Bergstrom * This is normally called from the host code's worker thread, but can be 2606579324aSTerje Bergstrom * called manually if necessary. 2616579324aSTerje Bergstrom * Must be called with the cdma lock held. 2626579324aSTerje Bergstrom */ 2636579324aSTerje Bergstrom static void update_cdma_locked(struct host1x_cdma *cdma) 2646579324aSTerje Bergstrom { 2656579324aSTerje Bergstrom bool signal = false; 2666579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 2676579324aSTerje Bergstrom struct host1x_job *job, *n; 2686579324aSTerje Bergstrom 2696579324aSTerje Bergstrom /* If CDMA is stopped, queue is cleared and we can return */ 2706579324aSTerje Bergstrom if (!cdma->running) 2716579324aSTerje Bergstrom return; 2726579324aSTerje Bergstrom 2736579324aSTerje Bergstrom /* 2746579324aSTerje Bergstrom * Walk the sync queue, reading the sync point registers as necessary, 2756579324aSTerje Bergstrom * to consume as many sync queue entries as possible without blocking 2766579324aSTerje Bergstrom */ 2776579324aSTerje Bergstrom list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { 2786579324aSTerje Bergstrom struct host1x_syncpt *sp = 2796579324aSTerje Bergstrom host1x_syncpt_get(host1x, job->syncpt_id); 2806579324aSTerje Bergstrom 2816579324aSTerje Bergstrom /* Check whether this syncpt has completed, and bail if not */ 2826579324aSTerje Bergstrom if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) { 2836579324aSTerje Bergstrom /* Start timer on next pending syncpt */ 2846579324aSTerje Bergstrom if (job->timeout) 2856579324aSTerje Bergstrom cdma_start_timer_locked(cdma, job); 2860b8070d1SThierry Reding 2876579324aSTerje Bergstrom break; 2886579324aSTerje Bergstrom } 2896579324aSTerje Bergstrom 2906579324aSTerje Bergstrom /* Cancel timeout, when a buffer completes */ 2916579324aSTerje Bergstrom if (cdma->timeout.client) 2926579324aSTerje Bergstrom stop_cdma_timer_locked(cdma); 2936579324aSTerje Bergstrom 2946579324aSTerje Bergstrom /* Unpin the memory */ 2956579324aSTerje Bergstrom host1x_job_unpin(job); 2966579324aSTerje Bergstrom 2976579324aSTerje Bergstrom /* Pop push buffer slots */ 2986579324aSTerje Bergstrom if (job->num_slots) { 2996579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 3006df633d0SThierry Reding 3016579324aSTerje Bergstrom host1x_pushbuffer_pop(pb, job->num_slots); 3020b8070d1SThierry Reding 3036579324aSTerje Bergstrom if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) 3046579324aSTerje Bergstrom signal = true; 3056579324aSTerje Bergstrom } 3066579324aSTerje Bergstrom 3076579324aSTerje Bergstrom list_del(&job->list); 3086579324aSTerje Bergstrom host1x_job_put(job); 3096579324aSTerje Bergstrom } 3106579324aSTerje Bergstrom 3116579324aSTerje Bergstrom if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && 3126579324aSTerje Bergstrom list_empty(&cdma->sync_queue)) 3136579324aSTerje Bergstrom signal = true; 3146579324aSTerje Bergstrom 3156579324aSTerje Bergstrom if (signal) { 3166579324aSTerje Bergstrom cdma->event = CDMA_EVENT_NONE; 3176579324aSTerje Bergstrom up(&cdma->sem); 3186579324aSTerje Bergstrom } 3196579324aSTerje Bergstrom } 3206579324aSTerje Bergstrom 3216579324aSTerje Bergstrom void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, 3226579324aSTerje Bergstrom struct device *dev) 3236579324aSTerje Bergstrom { 3246579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 3250b8070d1SThierry Reding u32 restart_addr, syncpt_incrs, syncpt_val; 3260b8070d1SThierry Reding struct host1x_job *job = NULL; 3276579324aSTerje Bergstrom 3286579324aSTerje Bergstrom syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 3296579324aSTerje Bergstrom 3306579324aSTerje Bergstrom dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", 3316579324aSTerje Bergstrom __func__, syncpt_val); 3326579324aSTerje Bergstrom 3336579324aSTerje Bergstrom /* 3346579324aSTerje Bergstrom * Move the sync_queue read pointer to the first entry that hasn't 3356579324aSTerje Bergstrom * completed based on the current HW syncpt value. It's likely there 3366579324aSTerje Bergstrom * won't be any (i.e. we're still at the head), but covers the case 3376579324aSTerje Bergstrom * where a syncpt incr happens just prior/during the teardown. 3386579324aSTerje Bergstrom */ 3396579324aSTerje Bergstrom 3406579324aSTerje Bergstrom dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", 3416579324aSTerje Bergstrom __func__); 3426579324aSTerje Bergstrom 3436579324aSTerje Bergstrom list_for_each_entry(job, &cdma->sync_queue, list) { 3446579324aSTerje Bergstrom if (syncpt_val < job->syncpt_end) 3456579324aSTerje Bergstrom break; 3466579324aSTerje Bergstrom 3476579324aSTerje Bergstrom host1x_job_dump(dev, job); 3486579324aSTerje Bergstrom } 3496579324aSTerje Bergstrom 3506579324aSTerje Bergstrom /* 3516579324aSTerje Bergstrom * Walk the sync_queue, first incrementing with the CPU syncpts that 3526579324aSTerje Bergstrom * are partially executed (the first buffer) or fully skipped while 3536579324aSTerje Bergstrom * still in the current context (slots are also NOP-ed). 3546579324aSTerje Bergstrom * 3556579324aSTerje Bergstrom * At the point contexts are interleaved, syncpt increments must be 3566579324aSTerje Bergstrom * done inline with the pushbuffer from a GATHER buffer to maintain 3576579324aSTerje Bergstrom * the order (slots are modified to be a GATHER of syncpt incrs). 3586579324aSTerje Bergstrom * 3596579324aSTerje Bergstrom * Note: save in restart_addr the location where the timed out buffer 3606579324aSTerje Bergstrom * started in the PB, so we can start the refetch from there (with the 3616579324aSTerje Bergstrom * modified NOP-ed PB slots). This lets things appear to have completed 3626579324aSTerje Bergstrom * properly for this buffer and resources are freed. 3636579324aSTerje Bergstrom */ 3646579324aSTerje Bergstrom 3656579324aSTerje Bergstrom dev_dbg(dev, "%s: perform CPU incr on pending same ctx buffers\n", 3666579324aSTerje Bergstrom __func__); 3676579324aSTerje Bergstrom 3686579324aSTerje Bergstrom if (!list_empty(&cdma->sync_queue)) 3696579324aSTerje Bergstrom restart_addr = job->first_get; 3706579324aSTerje Bergstrom else 3716579324aSTerje Bergstrom restart_addr = cdma->last_pos; 3726579324aSTerje Bergstrom 3736579324aSTerje Bergstrom /* do CPU increments as long as this context continues */ 3746579324aSTerje Bergstrom list_for_each_entry_from(job, &cdma->sync_queue, list) { 3756579324aSTerje Bergstrom /* different context, gets us out of this loop */ 3766579324aSTerje Bergstrom if (job->client != cdma->timeout.client) 3776579324aSTerje Bergstrom break; 3786579324aSTerje Bergstrom 3796579324aSTerje Bergstrom /* won't need a timeout when replayed */ 3806579324aSTerje Bergstrom job->timeout = 0; 3816579324aSTerje Bergstrom 3826579324aSTerje Bergstrom syncpt_incrs = job->syncpt_end - syncpt_val; 3836579324aSTerje Bergstrom dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); 3846579324aSTerje Bergstrom 3856579324aSTerje Bergstrom host1x_job_dump(dev, job); 3866579324aSTerje Bergstrom 3876579324aSTerje Bergstrom /* safe to use CPU to incr syncpts */ 3886579324aSTerje Bergstrom host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, 3896579324aSTerje Bergstrom syncpt_incrs, job->syncpt_end, 3906579324aSTerje Bergstrom job->num_slots); 3916579324aSTerje Bergstrom 3926579324aSTerje Bergstrom syncpt_val += syncpt_incrs; 3936579324aSTerje Bergstrom } 3946579324aSTerje Bergstrom 3956df633d0SThierry Reding /* 3966df633d0SThierry Reding * The following sumbits from the same client may be dependent on the 3976579324aSTerje Bergstrom * failed submit and therefore they may fail. Force a small timeout 3986df633d0SThierry Reding * to make the queue cleanup faster. 3996df633d0SThierry Reding */ 4006579324aSTerje Bergstrom 4016579324aSTerje Bergstrom list_for_each_entry_from(job, &cdma->sync_queue, list) 4026579324aSTerje Bergstrom if (job->client == cdma->timeout.client) 4036579324aSTerje Bergstrom job->timeout = min_t(unsigned int, job->timeout, 500); 4046579324aSTerje Bergstrom 4056579324aSTerje Bergstrom dev_dbg(dev, "%s: finished sync_queue modification\n", __func__); 4066579324aSTerje Bergstrom 4076579324aSTerje Bergstrom /* roll back DMAGET and start up channel again */ 4086579324aSTerje Bergstrom host1x_hw_cdma_resume(host1x, cdma, restart_addr); 4096579324aSTerje Bergstrom } 4106579324aSTerje Bergstrom 4116579324aSTerje Bergstrom /* 4126579324aSTerje Bergstrom * Create a cdma 4136579324aSTerje Bergstrom */ 4146579324aSTerje Bergstrom int host1x_cdma_init(struct host1x_cdma *cdma) 4156579324aSTerje Bergstrom { 4166579324aSTerje Bergstrom int err; 4176579324aSTerje Bergstrom 4186579324aSTerje Bergstrom mutex_init(&cdma->lock); 4196579324aSTerje Bergstrom sema_init(&cdma->sem, 0); 4206579324aSTerje Bergstrom 4216579324aSTerje Bergstrom INIT_LIST_HEAD(&cdma->sync_queue); 4226579324aSTerje Bergstrom 4236579324aSTerje Bergstrom cdma->event = CDMA_EVENT_NONE; 4246579324aSTerje Bergstrom cdma->running = false; 4256579324aSTerje Bergstrom cdma->torndown = false; 4266579324aSTerje Bergstrom 4276579324aSTerje Bergstrom err = host1x_pushbuffer_init(&cdma->push_buffer); 4286579324aSTerje Bergstrom if (err) 4296579324aSTerje Bergstrom return err; 4300b8070d1SThierry Reding 4316579324aSTerje Bergstrom return 0; 4326579324aSTerje Bergstrom } 4336579324aSTerje Bergstrom 4346579324aSTerje Bergstrom /* 4356579324aSTerje Bergstrom * Destroy a cdma 4366579324aSTerje Bergstrom */ 4376579324aSTerje Bergstrom int host1x_cdma_deinit(struct host1x_cdma *cdma) 4386579324aSTerje Bergstrom { 4396579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 4406579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 4416579324aSTerje Bergstrom 4426579324aSTerje Bergstrom if (cdma->running) { 4436579324aSTerje Bergstrom pr_warn("%s: CDMA still running\n", __func__); 4446579324aSTerje Bergstrom return -EBUSY; 4456579324aSTerje Bergstrom } 4466579324aSTerje Bergstrom 4476579324aSTerje Bergstrom host1x_pushbuffer_destroy(pb); 4486579324aSTerje Bergstrom host1x_hw_cdma_timeout_destroy(host1x, cdma); 4496579324aSTerje Bergstrom 4506579324aSTerje Bergstrom return 0; 4516579324aSTerje Bergstrom } 4526579324aSTerje Bergstrom 4536579324aSTerje Bergstrom /* 4546579324aSTerje Bergstrom * Begin a cdma submit 4556579324aSTerje Bergstrom */ 4566579324aSTerje Bergstrom int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job) 4576579324aSTerje Bergstrom { 4586579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 4596579324aSTerje Bergstrom 4606579324aSTerje Bergstrom mutex_lock(&cdma->lock); 4616579324aSTerje Bergstrom 4626579324aSTerje Bergstrom if (job->timeout) { 4636579324aSTerje Bergstrom /* init state on first submit with timeout value */ 4646579324aSTerje Bergstrom if (!cdma->timeout.initialized) { 4656579324aSTerje Bergstrom int err; 4666df633d0SThierry Reding 4676579324aSTerje Bergstrom err = host1x_hw_cdma_timeout_init(host1x, cdma, 4686579324aSTerje Bergstrom job->syncpt_id); 4696579324aSTerje Bergstrom if (err) { 4706579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 4716579324aSTerje Bergstrom return err; 4726579324aSTerje Bergstrom } 4736579324aSTerje Bergstrom } 4746579324aSTerje Bergstrom } 4750b8070d1SThierry Reding 4766579324aSTerje Bergstrom if (!cdma->running) 4776579324aSTerje Bergstrom host1x_hw_cdma_start(host1x, cdma); 4786579324aSTerje Bergstrom 4796579324aSTerje Bergstrom cdma->slots_free = 0; 4806579324aSTerje Bergstrom cdma->slots_used = 0; 4816579324aSTerje Bergstrom cdma->first_get = cdma->push_buffer.pos; 4826579324aSTerje Bergstrom 4836579324aSTerje Bergstrom trace_host1x_cdma_begin(dev_name(job->channel->dev)); 4846579324aSTerje Bergstrom return 0; 4856579324aSTerje Bergstrom } 4866579324aSTerje Bergstrom 4876579324aSTerje Bergstrom /* 4886579324aSTerje Bergstrom * Push two words into a push buffer slot 4896579324aSTerje Bergstrom * Blocks as necessary if the push buffer is full. 4906579324aSTerje Bergstrom */ 4916579324aSTerje Bergstrom void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2) 4926579324aSTerje Bergstrom { 4936579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 4946579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 4956579324aSTerje Bergstrom u32 slots_free = cdma->slots_free; 4966579324aSTerje Bergstrom 4976236451dSTerje Bergstrom if (host1x_debug_trace_cmdbuf) 4986236451dSTerje Bergstrom trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev), 4996236451dSTerje Bergstrom op1, op2); 5006236451dSTerje Bergstrom 5016579324aSTerje Bergstrom if (slots_free == 0) { 5026579324aSTerje Bergstrom host1x_hw_cdma_flush(host1x, cdma); 5036579324aSTerje Bergstrom slots_free = host1x_cdma_wait_locked(cdma, 5046579324aSTerje Bergstrom CDMA_EVENT_PUSH_BUFFER_SPACE); 5056579324aSTerje Bergstrom } 5060b8070d1SThierry Reding 5076579324aSTerje Bergstrom cdma->slots_free = slots_free - 1; 5086579324aSTerje Bergstrom cdma->slots_used++; 5096579324aSTerje Bergstrom host1x_pushbuffer_push(pb, op1, op2); 5106579324aSTerje Bergstrom } 5116579324aSTerje Bergstrom 5126579324aSTerje Bergstrom /* 5136579324aSTerje Bergstrom * End a cdma submit 5146579324aSTerje Bergstrom * Kick off DMA, add job to the sync queue, and a number of slots to be freed 5156579324aSTerje Bergstrom * from the pushbuffer. The handles for a submit must all be pinned at the same 5166579324aSTerje Bergstrom * time, but they can be unpinned in smaller chunks. 5176579324aSTerje Bergstrom */ 5186579324aSTerje Bergstrom void host1x_cdma_end(struct host1x_cdma *cdma, 5196579324aSTerje Bergstrom struct host1x_job *job) 5206579324aSTerje Bergstrom { 5216579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 5226579324aSTerje Bergstrom bool idle = list_empty(&cdma->sync_queue); 5236579324aSTerje Bergstrom 5246579324aSTerje Bergstrom host1x_hw_cdma_flush(host1x, cdma); 5256579324aSTerje Bergstrom 5266579324aSTerje Bergstrom job->first_get = cdma->first_get; 5276579324aSTerje Bergstrom job->num_slots = cdma->slots_used; 5286579324aSTerje Bergstrom host1x_job_get(job); 5296579324aSTerje Bergstrom list_add_tail(&job->list, &cdma->sync_queue); 5306579324aSTerje Bergstrom 5316579324aSTerje Bergstrom /* start timer on idle -> active transitions */ 5326579324aSTerje Bergstrom if (job->timeout && idle) 5336579324aSTerje Bergstrom cdma_start_timer_locked(cdma, job); 5346579324aSTerje Bergstrom 5356579324aSTerje Bergstrom trace_host1x_cdma_end(dev_name(job->channel->dev)); 5366579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 5376579324aSTerje Bergstrom } 5386579324aSTerje Bergstrom 5396579324aSTerje Bergstrom /* 5406579324aSTerje Bergstrom * Update cdma state according to current sync point values 5416579324aSTerje Bergstrom */ 5426579324aSTerje Bergstrom void host1x_cdma_update(struct host1x_cdma *cdma) 5436579324aSTerje Bergstrom { 5446579324aSTerje Bergstrom mutex_lock(&cdma->lock); 5456579324aSTerje Bergstrom update_cdma_locked(cdma); 5466579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 5476579324aSTerje Bergstrom } 548