16579324aSTerje Bergstrom /* 26579324aSTerje Bergstrom * Tegra host1x Command DMA 36579324aSTerje Bergstrom * 46579324aSTerje Bergstrom * Copyright (c) 2010-2013, NVIDIA Corporation. 56579324aSTerje Bergstrom * 66579324aSTerje Bergstrom * This program is free software; you can redistribute it and/or modify it 76579324aSTerje Bergstrom * under the terms and conditions of the GNU General Public License, 86579324aSTerje Bergstrom * version 2, as published by the Free Software Foundation. 96579324aSTerje Bergstrom * 106579324aSTerje Bergstrom * This program is distributed in the hope it will be useful, but WITHOUT 116579324aSTerje Bergstrom * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 126579324aSTerje Bergstrom * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 136579324aSTerje Bergstrom * more details. 146579324aSTerje Bergstrom * 156579324aSTerje Bergstrom * You should have received a copy of the GNU General Public License 166579324aSTerje Bergstrom * along with this program. If not, see <http://www.gnu.org/licenses/>. 176579324aSTerje Bergstrom */ 186579324aSTerje Bergstrom 196579324aSTerje Bergstrom 206579324aSTerje Bergstrom #include <asm/cacheflush.h> 216579324aSTerje Bergstrom #include <linux/device.h> 226579324aSTerje Bergstrom #include <linux/dma-mapping.h> 2335d747a8SThierry Reding #include <linux/host1x.h> 246579324aSTerje Bergstrom #include <linux/interrupt.h> 256579324aSTerje Bergstrom #include <linux/kernel.h> 266579324aSTerje Bergstrom #include <linux/kfifo.h> 276579324aSTerje Bergstrom #include <linux/slab.h> 286579324aSTerje Bergstrom #include <trace/events/host1x.h> 296579324aSTerje Bergstrom 306579324aSTerje Bergstrom #include "cdma.h" 316579324aSTerje Bergstrom #include "channel.h" 326579324aSTerje Bergstrom #include "dev.h" 336579324aSTerje Bergstrom #include "debug.h" 346579324aSTerje Bergstrom #include "job.h" 356579324aSTerje Bergstrom 366579324aSTerje Bergstrom /* 376579324aSTerje Bergstrom * push_buffer 386579324aSTerje Bergstrom * 396579324aSTerje Bergstrom * The push buffer is a circular array of words to be fetched by command DMA. 406579324aSTerje Bergstrom * Note that it works slightly differently to the sync queue; fence == pos 416579324aSTerje Bergstrom * means that the push buffer is full, not empty. 426579324aSTerje Bergstrom */ 436579324aSTerje Bergstrom 44e1f338c0SThierry Reding /* 45e1f338c0SThierry Reding * Typically the commands written into the push buffer are a pair of words. We 46e1f338c0SThierry Reding * use slots to represent each of these pairs and to simplify things. Note the 47e1f338c0SThierry Reding * strange number of slots allocated here. 512 slots will fit exactly within a 48e1f338c0SThierry Reding * single memory page. We also need one additional word at the end of the push 49e1f338c0SThierry Reding * buffer for the RESTART opcode that will instruct the CDMA to jump back to 50e1f338c0SThierry Reding * the beginning of the push buffer. With 512 slots, this means that we'll use 51e1f338c0SThierry Reding * 2 memory pages and waste 4092 bytes of the second page that will never be 52e1f338c0SThierry Reding * used. 53e1f338c0SThierry Reding */ 54e1f338c0SThierry Reding #define HOST1X_PUSHBUFFER_SLOTS 511 556579324aSTerje Bergstrom 566579324aSTerje Bergstrom /* 576579324aSTerje Bergstrom * Clean up push buffer resources 586579324aSTerje Bergstrom */ 596579324aSTerje Bergstrom static void host1x_pushbuffer_destroy(struct push_buffer *pb) 606579324aSTerje Bergstrom { 616579324aSTerje Bergstrom struct host1x_cdma *cdma = pb_to_cdma(pb); 626579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 636579324aSTerje Bergstrom 642f8a6da8SEmil Goode if (!pb->mapped) 65404bfb78SMikko Perttunen return; 66404bfb78SMikko Perttunen 67404bfb78SMikko Perttunen if (host1x->domain) { 68404bfb78SMikko Perttunen iommu_unmap(host1x->domain, pb->dma, pb->alloc_size); 69404bfb78SMikko Perttunen free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma)); 70404bfb78SMikko Perttunen } 71404bfb78SMikko Perttunen 72404bfb78SMikko Perttunen dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys); 736579324aSTerje Bergstrom 746579324aSTerje Bergstrom pb->mapped = NULL; 756579324aSTerje Bergstrom pb->phys = 0; 766579324aSTerje Bergstrom } 776579324aSTerje Bergstrom 786579324aSTerje Bergstrom /* 796579324aSTerje Bergstrom * Init push buffer resources 806579324aSTerje Bergstrom */ 816579324aSTerje Bergstrom static int host1x_pushbuffer_init(struct push_buffer *pb) 826579324aSTerje Bergstrom { 836579324aSTerje Bergstrom struct host1x_cdma *cdma = pb_to_cdma(pb); 846579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 85404bfb78SMikko Perttunen struct iova *alloc; 86404bfb78SMikko Perttunen u32 size; 87404bfb78SMikko Perttunen int err; 886579324aSTerje Bergstrom 896579324aSTerje Bergstrom pb->mapped = NULL; 906579324aSTerje Bergstrom pb->phys = 0; 91404bfb78SMikko Perttunen pb->size = HOST1X_PUSHBUFFER_SLOTS * 8; 92404bfb78SMikko Perttunen 93404bfb78SMikko Perttunen size = pb->size + 4; 946579324aSTerje Bergstrom 956579324aSTerje Bergstrom /* initialize buffer pointers */ 96404bfb78SMikko Perttunen pb->fence = pb->size - 8; 976579324aSTerje Bergstrom pb->pos = 0; 986579324aSTerje Bergstrom 99404bfb78SMikko Perttunen if (host1x->domain) { 100404bfb78SMikko Perttunen unsigned long shift; 101404bfb78SMikko Perttunen 102404bfb78SMikko Perttunen size = iova_align(&host1x->iova, size); 103404bfb78SMikko Perttunen 104404bfb78SMikko Perttunen pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, 105f6e45661SLuis R. Rodriguez GFP_KERNEL); 1066579324aSTerje Bergstrom if (!pb->mapped) 107404bfb78SMikko Perttunen return -ENOMEM; 108404bfb78SMikko Perttunen 109404bfb78SMikko Perttunen shift = iova_shift(&host1x->iova); 110404bfb78SMikko Perttunen alloc = alloc_iova(&host1x->iova, size >> shift, 111404bfb78SMikko Perttunen host1x->iova_end >> shift, true); 112404bfb78SMikko Perttunen if (!alloc) { 113404bfb78SMikko Perttunen err = -ENOMEM; 114404bfb78SMikko Perttunen goto iommu_free_mem; 115404bfb78SMikko Perttunen } 116404bfb78SMikko Perttunen 117404bfb78SMikko Perttunen pb->dma = iova_dma_addr(&host1x->iova, alloc); 118404bfb78SMikko Perttunen err = iommu_map(host1x->domain, pb->dma, pb->phys, size, 119404bfb78SMikko Perttunen IOMMU_READ); 120404bfb78SMikko Perttunen if (err) 121404bfb78SMikko Perttunen goto iommu_free_iova; 122404bfb78SMikko Perttunen } else { 123404bfb78SMikko Perttunen pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, 124404bfb78SMikko Perttunen GFP_KERNEL); 125404bfb78SMikko Perttunen if (!pb->mapped) 126404bfb78SMikko Perttunen return -ENOMEM; 127404bfb78SMikko Perttunen 128404bfb78SMikko Perttunen pb->dma = pb->phys; 129404bfb78SMikko Perttunen } 130404bfb78SMikko Perttunen 131404bfb78SMikko Perttunen pb->alloc_size = size; 1326579324aSTerje Bergstrom 1336579324aSTerje Bergstrom host1x_hw_pushbuffer_init(host1x, pb); 1346579324aSTerje Bergstrom 1356579324aSTerje Bergstrom return 0; 1366579324aSTerje Bergstrom 137404bfb78SMikko Perttunen iommu_free_iova: 138404bfb78SMikko Perttunen __free_iova(&host1x->iova, alloc); 139404bfb78SMikko Perttunen iommu_free_mem: 14027db6a00SDmitry Osipenko dma_free_wc(host1x->dev, size, pb->mapped, pb->phys); 141404bfb78SMikko Perttunen 142404bfb78SMikko Perttunen return err; 1436579324aSTerje Bergstrom } 1446579324aSTerje Bergstrom 1456579324aSTerje Bergstrom /* 1466579324aSTerje Bergstrom * Push two words to the push buffer 1476579324aSTerje Bergstrom * Caller must ensure push buffer is not full 1486579324aSTerje Bergstrom */ 1496579324aSTerje Bergstrom static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2) 1506579324aSTerje Bergstrom { 151ebb2475cSThierry Reding u32 *p = (u32 *)((void *)pb->mapped + pb->pos); 152ebb2475cSThierry Reding 153ebb2475cSThierry Reding WARN_ON(pb->pos == pb->fence); 1546579324aSTerje Bergstrom *(p++) = op1; 1556579324aSTerje Bergstrom *(p++) = op2; 156e1f338c0SThierry Reding pb->pos += 8; 157e1f338c0SThierry Reding 158e1f338c0SThierry Reding if (pb->pos >= pb->size) 159e1f338c0SThierry Reding pb->pos -= pb->size; 1606579324aSTerje Bergstrom } 1616579324aSTerje Bergstrom 1626579324aSTerje Bergstrom /* 1636579324aSTerje Bergstrom * Pop a number of two word slots from the push buffer 1646579324aSTerje Bergstrom * Caller must ensure push buffer is not empty 1656579324aSTerje Bergstrom */ 1666579324aSTerje Bergstrom static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots) 1676579324aSTerje Bergstrom { 1686579324aSTerje Bergstrom /* Advance the next write position */ 169e1f338c0SThierry Reding pb->fence += slots * 8; 170e1f338c0SThierry Reding 171e1f338c0SThierry Reding if (pb->fence >= pb->size) 172e1f338c0SThierry Reding pb->fence -= pb->size; 1736579324aSTerje Bergstrom } 1746579324aSTerje Bergstrom 1756579324aSTerje Bergstrom /* 1766579324aSTerje Bergstrom * Return the number of two word slots free in the push buffer 1776579324aSTerje Bergstrom */ 1786579324aSTerje Bergstrom static u32 host1x_pushbuffer_space(struct push_buffer *pb) 1796579324aSTerje Bergstrom { 180e1f338c0SThierry Reding unsigned int fence = pb->fence; 181e1f338c0SThierry Reding 182e1f338c0SThierry Reding if (pb->fence < pb->pos) 183e1f338c0SThierry Reding fence += pb->size; 184e1f338c0SThierry Reding 185e1f338c0SThierry Reding return (fence - pb->pos) / 8; 1866579324aSTerje Bergstrom } 1876579324aSTerje Bergstrom 1886579324aSTerje Bergstrom /* 1896579324aSTerje Bergstrom * Sleep (if necessary) until the requested event happens 1906579324aSTerje Bergstrom * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. 1916579324aSTerje Bergstrom * - Returns 1 1926579324aSTerje Bergstrom * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer 1936579324aSTerje Bergstrom * - Return the amount of space (> 0) 1946579324aSTerje Bergstrom * Must be called with the cdma lock held. 1956579324aSTerje Bergstrom */ 1966579324aSTerje Bergstrom unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, 1976579324aSTerje Bergstrom enum cdma_event event) 1986579324aSTerje Bergstrom { 1996579324aSTerje Bergstrom for (;;) { 2000b8070d1SThierry Reding struct push_buffer *pb = &cdma->push_buffer; 2016579324aSTerje Bergstrom unsigned int space; 2026579324aSTerje Bergstrom 2030b8070d1SThierry Reding switch (event) { 2040b8070d1SThierry Reding case CDMA_EVENT_SYNC_QUEUE_EMPTY: 2056579324aSTerje Bergstrom space = list_empty(&cdma->sync_queue) ? 1 : 0; 2060b8070d1SThierry Reding break; 2070b8070d1SThierry Reding 2080b8070d1SThierry Reding case CDMA_EVENT_PUSH_BUFFER_SPACE: 2096579324aSTerje Bergstrom space = host1x_pushbuffer_space(pb); 2100b8070d1SThierry Reding break; 2110b8070d1SThierry Reding 2120b8070d1SThierry Reding default: 2136579324aSTerje Bergstrom WARN_ON(1); 2146579324aSTerje Bergstrom return -EINVAL; 2156579324aSTerje Bergstrom } 2166579324aSTerje Bergstrom 2176579324aSTerje Bergstrom if (space) 2186579324aSTerje Bergstrom return space; 2196579324aSTerje Bergstrom 2206579324aSTerje Bergstrom trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), 2216579324aSTerje Bergstrom event); 2226579324aSTerje Bergstrom 2236579324aSTerje Bergstrom /* If somebody has managed to already start waiting, yield */ 2246579324aSTerje Bergstrom if (cdma->event != CDMA_EVENT_NONE) { 2256579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 2266579324aSTerje Bergstrom schedule(); 2276579324aSTerje Bergstrom mutex_lock(&cdma->lock); 2286579324aSTerje Bergstrom continue; 2296579324aSTerje Bergstrom } 2300b8070d1SThierry Reding 2316579324aSTerje Bergstrom cdma->event = event; 2326579324aSTerje Bergstrom 2336579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 2340747a672SArnd Bergmann wait_for_completion(&cdma->complete); 2356579324aSTerje Bergstrom mutex_lock(&cdma->lock); 2366579324aSTerje Bergstrom } 2370b8070d1SThierry Reding 2386579324aSTerje Bergstrom return 0; 2396579324aSTerje Bergstrom } 2406579324aSTerje Bergstrom 2416579324aSTerje Bergstrom /* 2425a5fccbdSThierry Reding * Sleep (if necessary) until the push buffer has enough free space. 2435a5fccbdSThierry Reding * 2445a5fccbdSThierry Reding * Must be called with the cdma lock held. 2455a5fccbdSThierry Reding */ 2465a5fccbdSThierry Reding int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x, 2475a5fccbdSThierry Reding struct host1x_cdma *cdma, 2485a5fccbdSThierry Reding unsigned int needed) 2495a5fccbdSThierry Reding { 2505a5fccbdSThierry Reding while (true) { 2515a5fccbdSThierry Reding struct push_buffer *pb = &cdma->push_buffer; 2525a5fccbdSThierry Reding unsigned int space; 2535a5fccbdSThierry Reding 2545a5fccbdSThierry Reding space = host1x_pushbuffer_space(pb); 2555a5fccbdSThierry Reding if (space >= needed) 2565a5fccbdSThierry Reding break; 2575a5fccbdSThierry Reding 2585a5fccbdSThierry Reding trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), 2595a5fccbdSThierry Reding CDMA_EVENT_PUSH_BUFFER_SPACE); 2605a5fccbdSThierry Reding 2615a5fccbdSThierry Reding host1x_hw_cdma_flush(host1x, cdma); 2625a5fccbdSThierry Reding 2635a5fccbdSThierry Reding /* If somebody has managed to already start waiting, yield */ 2645a5fccbdSThierry Reding if (cdma->event != CDMA_EVENT_NONE) { 2655a5fccbdSThierry Reding mutex_unlock(&cdma->lock); 2665a5fccbdSThierry Reding schedule(); 2675a5fccbdSThierry Reding mutex_lock(&cdma->lock); 2685a5fccbdSThierry Reding continue; 2695a5fccbdSThierry Reding } 2705a5fccbdSThierry Reding 2715a5fccbdSThierry Reding cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE; 2725a5fccbdSThierry Reding 2735a5fccbdSThierry Reding mutex_unlock(&cdma->lock); 2745a5fccbdSThierry Reding wait_for_completion(&cdma->complete); 2755a5fccbdSThierry Reding mutex_lock(&cdma->lock); 2765a5fccbdSThierry Reding } 2775a5fccbdSThierry Reding 2785a5fccbdSThierry Reding return 0; 2795a5fccbdSThierry Reding } 2805a5fccbdSThierry Reding /* 2816579324aSTerje Bergstrom * Start timer that tracks the time spent by the job. 2826579324aSTerje Bergstrom * Must be called with the cdma lock held. 2836579324aSTerje Bergstrom */ 2846579324aSTerje Bergstrom static void cdma_start_timer_locked(struct host1x_cdma *cdma, 2856579324aSTerje Bergstrom struct host1x_job *job) 2866579324aSTerje Bergstrom { 2876579324aSTerje Bergstrom struct host1x *host = cdma_to_host1x(cdma); 2886579324aSTerje Bergstrom 2896579324aSTerje Bergstrom if (cdma->timeout.client) { 2906579324aSTerje Bergstrom /* timer already started */ 2916579324aSTerje Bergstrom return; 2926579324aSTerje Bergstrom } 2936579324aSTerje Bergstrom 2946579324aSTerje Bergstrom cdma->timeout.client = job->client; 2956579324aSTerje Bergstrom cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id); 2966579324aSTerje Bergstrom cdma->timeout.syncpt_val = job->syncpt_end; 2976579324aSTerje Bergstrom cdma->timeout.start_ktime = ktime_get(); 2986579324aSTerje Bergstrom 2996579324aSTerje Bergstrom schedule_delayed_work(&cdma->timeout.wq, 3006579324aSTerje Bergstrom msecs_to_jiffies(job->timeout)); 3016579324aSTerje Bergstrom } 3026579324aSTerje Bergstrom 3036579324aSTerje Bergstrom /* 3046579324aSTerje Bergstrom * Stop timer when a buffer submission completes. 3056579324aSTerje Bergstrom * Must be called with the cdma lock held. 3066579324aSTerje Bergstrom */ 3076579324aSTerje Bergstrom static void stop_cdma_timer_locked(struct host1x_cdma *cdma) 3086579324aSTerje Bergstrom { 3096579324aSTerje Bergstrom cancel_delayed_work(&cdma->timeout.wq); 310bf3d41ccSThierry Reding cdma->timeout.client = NULL; 3116579324aSTerje Bergstrom } 3126579324aSTerje Bergstrom 3136579324aSTerje Bergstrom /* 3146579324aSTerje Bergstrom * For all sync queue entries that have already finished according to the 3156579324aSTerje Bergstrom * current sync point registers: 3166579324aSTerje Bergstrom * - unpin & unref their mems 3176579324aSTerje Bergstrom * - pop their push buffer slots 3186579324aSTerje Bergstrom * - remove them from the sync queue 3196579324aSTerje Bergstrom * This is normally called from the host code's worker thread, but can be 3206579324aSTerje Bergstrom * called manually if necessary. 3216579324aSTerje Bergstrom * Must be called with the cdma lock held. 3226579324aSTerje Bergstrom */ 3236579324aSTerje Bergstrom static void update_cdma_locked(struct host1x_cdma *cdma) 3246579324aSTerje Bergstrom { 3256579324aSTerje Bergstrom bool signal = false; 3266579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 3276579324aSTerje Bergstrom struct host1x_job *job, *n; 3286579324aSTerje Bergstrom 3296579324aSTerje Bergstrom /* If CDMA is stopped, queue is cleared and we can return */ 3306579324aSTerje Bergstrom if (!cdma->running) 3316579324aSTerje Bergstrom return; 3326579324aSTerje Bergstrom 3336579324aSTerje Bergstrom /* 3346579324aSTerje Bergstrom * Walk the sync queue, reading the sync point registers as necessary, 3356579324aSTerje Bergstrom * to consume as many sync queue entries as possible without blocking 3366579324aSTerje Bergstrom */ 3376579324aSTerje Bergstrom list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { 3386579324aSTerje Bergstrom struct host1x_syncpt *sp = 3396579324aSTerje Bergstrom host1x_syncpt_get(host1x, job->syncpt_id); 3406579324aSTerje Bergstrom 3416579324aSTerje Bergstrom /* Check whether this syncpt has completed, and bail if not */ 3426579324aSTerje Bergstrom if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) { 3436579324aSTerje Bergstrom /* Start timer on next pending syncpt */ 3446579324aSTerje Bergstrom if (job->timeout) 3456579324aSTerje Bergstrom cdma_start_timer_locked(cdma, job); 3460b8070d1SThierry Reding 3476579324aSTerje Bergstrom break; 3486579324aSTerje Bergstrom } 3496579324aSTerje Bergstrom 3506579324aSTerje Bergstrom /* Cancel timeout, when a buffer completes */ 3516579324aSTerje Bergstrom if (cdma->timeout.client) 3526579324aSTerje Bergstrom stop_cdma_timer_locked(cdma); 3536579324aSTerje Bergstrom 3546579324aSTerje Bergstrom /* Unpin the memory */ 3556579324aSTerje Bergstrom host1x_job_unpin(job); 3566579324aSTerje Bergstrom 3576579324aSTerje Bergstrom /* Pop push buffer slots */ 3586579324aSTerje Bergstrom if (job->num_slots) { 3596579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 3606df633d0SThierry Reding 3616579324aSTerje Bergstrom host1x_pushbuffer_pop(pb, job->num_slots); 3620b8070d1SThierry Reding 3636579324aSTerje Bergstrom if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) 3646579324aSTerje Bergstrom signal = true; 3656579324aSTerje Bergstrom } 3666579324aSTerje Bergstrom 3676579324aSTerje Bergstrom list_del(&job->list); 3686579324aSTerje Bergstrom host1x_job_put(job); 3696579324aSTerje Bergstrom } 3706579324aSTerje Bergstrom 3716579324aSTerje Bergstrom if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && 3726579324aSTerje Bergstrom list_empty(&cdma->sync_queue)) 3736579324aSTerje Bergstrom signal = true; 3746579324aSTerje Bergstrom 3756579324aSTerje Bergstrom if (signal) { 3766579324aSTerje Bergstrom cdma->event = CDMA_EVENT_NONE; 3770747a672SArnd Bergmann complete(&cdma->complete); 3786579324aSTerje Bergstrom } 3796579324aSTerje Bergstrom } 3806579324aSTerje Bergstrom 3816579324aSTerje Bergstrom void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, 3826579324aSTerje Bergstrom struct device *dev) 3836579324aSTerje Bergstrom { 3846579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 3850b8070d1SThierry Reding u32 restart_addr, syncpt_incrs, syncpt_val; 38679930bafSDmitry Osipenko struct host1x_job *job, *next_job = NULL; 3876579324aSTerje Bergstrom 3886579324aSTerje Bergstrom syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); 3896579324aSTerje Bergstrom 3906579324aSTerje Bergstrom dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", 3916579324aSTerje Bergstrom __func__, syncpt_val); 3926579324aSTerje Bergstrom 3936579324aSTerje Bergstrom /* 3946579324aSTerje Bergstrom * Move the sync_queue read pointer to the first entry that hasn't 3956579324aSTerje Bergstrom * completed based on the current HW syncpt value. It's likely there 3966579324aSTerje Bergstrom * won't be any (i.e. we're still at the head), but covers the case 3976579324aSTerje Bergstrom * where a syncpt incr happens just prior/during the teardown. 3986579324aSTerje Bergstrom */ 3996579324aSTerje Bergstrom 4006579324aSTerje Bergstrom dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", 4016579324aSTerje Bergstrom __func__); 4026579324aSTerje Bergstrom 4036579324aSTerje Bergstrom list_for_each_entry(job, &cdma->sync_queue, list) { 40479930bafSDmitry Osipenko if (syncpt_val < job->syncpt_end) { 40579930bafSDmitry Osipenko 40679930bafSDmitry Osipenko if (!list_is_last(&job->list, &cdma->sync_queue)) 40779930bafSDmitry Osipenko next_job = list_next_entry(job, list); 40879930bafSDmitry Osipenko 4095d6f0436SDmitry Osipenko goto syncpt_incr; 41079930bafSDmitry Osipenko } 4116579324aSTerje Bergstrom 4126579324aSTerje Bergstrom host1x_job_dump(dev, job); 4136579324aSTerje Bergstrom } 4146579324aSTerje Bergstrom 4155d6f0436SDmitry Osipenko /* all jobs have been completed */ 4165d6f0436SDmitry Osipenko job = NULL; 4175d6f0436SDmitry Osipenko 4185d6f0436SDmitry Osipenko syncpt_incr: 4195d6f0436SDmitry Osipenko 4206579324aSTerje Bergstrom /* 421e8bad659SDmitry Osipenko * Increment with CPU the remaining syncpts of a partially executed job. 4226579324aSTerje Bergstrom * 42379930bafSDmitry Osipenko * CDMA will continue execution starting with the next job or will get 42479930bafSDmitry Osipenko * into idle state. 4256579324aSTerje Bergstrom */ 42679930bafSDmitry Osipenko if (next_job) 42779930bafSDmitry Osipenko restart_addr = next_job->first_get; 4286579324aSTerje Bergstrom else 4296579324aSTerje Bergstrom restart_addr = cdma->last_pos; 4306579324aSTerje Bergstrom 431e8bad659SDmitry Osipenko /* do CPU increments for the remaining syncpts */ 432e8bad659SDmitry Osipenko if (job) { 4335d6f0436SDmitry Osipenko dev_dbg(dev, "%s: perform CPU incr on pending buffers\n", 4345d6f0436SDmitry Osipenko __func__); 4355d6f0436SDmitry Osipenko 4366579324aSTerje Bergstrom /* won't need a timeout when replayed */ 4376579324aSTerje Bergstrom job->timeout = 0; 4386579324aSTerje Bergstrom 4396579324aSTerje Bergstrom syncpt_incrs = job->syncpt_end - syncpt_val; 4406579324aSTerje Bergstrom dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); 4416579324aSTerje Bergstrom 4426579324aSTerje Bergstrom host1x_job_dump(dev, job); 4436579324aSTerje Bergstrom 4446579324aSTerje Bergstrom /* safe to use CPU to incr syncpts */ 4456579324aSTerje Bergstrom host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, 4466579324aSTerje Bergstrom syncpt_incrs, job->syncpt_end, 4476579324aSTerje Bergstrom job->num_slots); 4486579324aSTerje Bergstrom 4495d6f0436SDmitry Osipenko dev_dbg(dev, "%s: finished sync_queue modification\n", 4505d6f0436SDmitry Osipenko __func__); 4515d6f0436SDmitry Osipenko } 4526579324aSTerje Bergstrom 4536579324aSTerje Bergstrom /* roll back DMAGET and start up channel again */ 4546579324aSTerje Bergstrom host1x_hw_cdma_resume(host1x, cdma, restart_addr); 4556579324aSTerje Bergstrom } 4566579324aSTerje Bergstrom 4576579324aSTerje Bergstrom /* 4586579324aSTerje Bergstrom * Create a cdma 4596579324aSTerje Bergstrom */ 4606579324aSTerje Bergstrom int host1x_cdma_init(struct host1x_cdma *cdma) 4616579324aSTerje Bergstrom { 4626579324aSTerje Bergstrom int err; 4636579324aSTerje Bergstrom 4646579324aSTerje Bergstrom mutex_init(&cdma->lock); 4650747a672SArnd Bergmann init_completion(&cdma->complete); 4666579324aSTerje Bergstrom 4676579324aSTerje Bergstrom INIT_LIST_HEAD(&cdma->sync_queue); 4686579324aSTerje Bergstrom 4696579324aSTerje Bergstrom cdma->event = CDMA_EVENT_NONE; 4706579324aSTerje Bergstrom cdma->running = false; 4716579324aSTerje Bergstrom cdma->torndown = false; 4726579324aSTerje Bergstrom 4736579324aSTerje Bergstrom err = host1x_pushbuffer_init(&cdma->push_buffer); 4746579324aSTerje Bergstrom if (err) 4756579324aSTerje Bergstrom return err; 4760b8070d1SThierry Reding 4776579324aSTerje Bergstrom return 0; 4786579324aSTerje Bergstrom } 4796579324aSTerje Bergstrom 4806579324aSTerje Bergstrom /* 4816579324aSTerje Bergstrom * Destroy a cdma 4826579324aSTerje Bergstrom */ 4836579324aSTerje Bergstrom int host1x_cdma_deinit(struct host1x_cdma *cdma) 4846579324aSTerje Bergstrom { 4856579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 4866579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 4876579324aSTerje Bergstrom 4886579324aSTerje Bergstrom if (cdma->running) { 4896579324aSTerje Bergstrom pr_warn("%s: CDMA still running\n", __func__); 4906579324aSTerje Bergstrom return -EBUSY; 4916579324aSTerje Bergstrom } 4926579324aSTerje Bergstrom 4936579324aSTerje Bergstrom host1x_pushbuffer_destroy(pb); 4946579324aSTerje Bergstrom host1x_hw_cdma_timeout_destroy(host1x, cdma); 4956579324aSTerje Bergstrom 4966579324aSTerje Bergstrom return 0; 4976579324aSTerje Bergstrom } 4986579324aSTerje Bergstrom 4996579324aSTerje Bergstrom /* 5006579324aSTerje Bergstrom * Begin a cdma submit 5016579324aSTerje Bergstrom */ 5026579324aSTerje Bergstrom int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job) 5036579324aSTerje Bergstrom { 5046579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 5056579324aSTerje Bergstrom 5066579324aSTerje Bergstrom mutex_lock(&cdma->lock); 5076579324aSTerje Bergstrom 5086579324aSTerje Bergstrom if (job->timeout) { 5096579324aSTerje Bergstrom /* init state on first submit with timeout value */ 5106579324aSTerje Bergstrom if (!cdma->timeout.initialized) { 5116579324aSTerje Bergstrom int err; 5126df633d0SThierry Reding 5136579324aSTerje Bergstrom err = host1x_hw_cdma_timeout_init(host1x, cdma, 5146579324aSTerje Bergstrom job->syncpt_id); 5156579324aSTerje Bergstrom if (err) { 5166579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 5176579324aSTerje Bergstrom return err; 5186579324aSTerje Bergstrom } 5196579324aSTerje Bergstrom } 5206579324aSTerje Bergstrom } 5210b8070d1SThierry Reding 5226579324aSTerje Bergstrom if (!cdma->running) 5236579324aSTerje Bergstrom host1x_hw_cdma_start(host1x, cdma); 5246579324aSTerje Bergstrom 5256579324aSTerje Bergstrom cdma->slots_free = 0; 5266579324aSTerje Bergstrom cdma->slots_used = 0; 5276579324aSTerje Bergstrom cdma->first_get = cdma->push_buffer.pos; 5286579324aSTerje Bergstrom 5296579324aSTerje Bergstrom trace_host1x_cdma_begin(dev_name(job->channel->dev)); 5306579324aSTerje Bergstrom return 0; 5316579324aSTerje Bergstrom } 5326579324aSTerje Bergstrom 5336579324aSTerje Bergstrom /* 5346579324aSTerje Bergstrom * Push two words into a push buffer slot 5356579324aSTerje Bergstrom * Blocks as necessary if the push buffer is full. 5366579324aSTerje Bergstrom */ 5376579324aSTerje Bergstrom void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2) 5386579324aSTerje Bergstrom { 5396579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 5406579324aSTerje Bergstrom struct push_buffer *pb = &cdma->push_buffer; 5416579324aSTerje Bergstrom u32 slots_free = cdma->slots_free; 5426579324aSTerje Bergstrom 5436236451dSTerje Bergstrom if (host1x_debug_trace_cmdbuf) 5446236451dSTerje Bergstrom trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev), 5456236451dSTerje Bergstrom op1, op2); 5466236451dSTerje Bergstrom 5476579324aSTerje Bergstrom if (slots_free == 0) { 5486579324aSTerje Bergstrom host1x_hw_cdma_flush(host1x, cdma); 5496579324aSTerje Bergstrom slots_free = host1x_cdma_wait_locked(cdma, 5506579324aSTerje Bergstrom CDMA_EVENT_PUSH_BUFFER_SPACE); 5516579324aSTerje Bergstrom } 5520b8070d1SThierry Reding 5536579324aSTerje Bergstrom cdma->slots_free = slots_free - 1; 5546579324aSTerje Bergstrom cdma->slots_used++; 5556579324aSTerje Bergstrom host1x_pushbuffer_push(pb, op1, op2); 5566579324aSTerje Bergstrom } 5576579324aSTerje Bergstrom 5586579324aSTerje Bergstrom /* 5595a5fccbdSThierry Reding * Push four words into two consecutive push buffer slots. Note that extra 5605a5fccbdSThierry Reding * care needs to be taken not to split the two slots across the end of the 5615a5fccbdSThierry Reding * push buffer. Otherwise the RESTART opcode at the end of the push buffer 5625a5fccbdSThierry Reding * that ensures processing will restart at the beginning will break up the 5635a5fccbdSThierry Reding * four words. 5645a5fccbdSThierry Reding * 5655a5fccbdSThierry Reding * Blocks as necessary if the push buffer is full. 5665a5fccbdSThierry Reding */ 5675a5fccbdSThierry Reding void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2, 5685a5fccbdSThierry Reding u32 op3, u32 op4) 5695a5fccbdSThierry Reding { 5705a5fccbdSThierry Reding struct host1x_channel *channel = cdma_to_channel(cdma); 5715a5fccbdSThierry Reding struct host1x *host1x = cdma_to_host1x(cdma); 5725a5fccbdSThierry Reding struct push_buffer *pb = &cdma->push_buffer; 5735a5fccbdSThierry Reding unsigned int needed = 2, extra = 0, i; 5745a5fccbdSThierry Reding unsigned int space = cdma->slots_free; 5755a5fccbdSThierry Reding 5765a5fccbdSThierry Reding if (host1x_debug_trace_cmdbuf) 5775a5fccbdSThierry Reding trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2, 5785a5fccbdSThierry Reding op3, op4); 5795a5fccbdSThierry Reding 5805a5fccbdSThierry Reding /* compute number of extra slots needed for padding */ 5815a5fccbdSThierry Reding if (pb->pos + 16 > pb->size) { 5825a5fccbdSThierry Reding extra = (pb->size - pb->pos) / 8; 5835a5fccbdSThierry Reding needed += extra; 5845a5fccbdSThierry Reding } 5855a5fccbdSThierry Reding 5865a5fccbdSThierry Reding host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed); 5875a5fccbdSThierry Reding space = host1x_pushbuffer_space(pb); 5885a5fccbdSThierry Reding 5895a5fccbdSThierry Reding cdma->slots_free = space - needed; 5905a5fccbdSThierry Reding cdma->slots_used += needed; 5915a5fccbdSThierry Reding 5925a5fccbdSThierry Reding /* 5935a5fccbdSThierry Reding * Note that we rely on the fact that this is only used to submit wide 5945a5fccbdSThierry Reding * gather opcodes, which consist of 3 words, and they are padded with 5955a5fccbdSThierry Reding * a NOP to avoid having to deal with fractional slots (a slot always 5965a5fccbdSThierry Reding * represents 2 words). The fourth opcode passed to this function will 5975a5fccbdSThierry Reding * therefore always be a NOP. 5985a5fccbdSThierry Reding * 5995a5fccbdSThierry Reding * This works around a slight ambiguity when it comes to opcodes. For 6005a5fccbdSThierry Reding * all current host1x incarnations the NOP opcode uses the exact same 6015a5fccbdSThierry Reding * encoding (0x20000000), so we could hard-code the value here, but a 6025a5fccbdSThierry Reding * new incarnation may change it and break that assumption. 6035a5fccbdSThierry Reding */ 6045a5fccbdSThierry Reding for (i = 0; i < extra; i++) 6055a5fccbdSThierry Reding host1x_pushbuffer_push(pb, op4, op4); 6065a5fccbdSThierry Reding 6075a5fccbdSThierry Reding host1x_pushbuffer_push(pb, op1, op2); 6085a5fccbdSThierry Reding host1x_pushbuffer_push(pb, op3, op4); 6095a5fccbdSThierry Reding } 6105a5fccbdSThierry Reding 6115a5fccbdSThierry Reding /* 6126579324aSTerje Bergstrom * End a cdma submit 6136579324aSTerje Bergstrom * Kick off DMA, add job to the sync queue, and a number of slots to be freed 6146579324aSTerje Bergstrom * from the pushbuffer. The handles for a submit must all be pinned at the same 6156579324aSTerje Bergstrom * time, but they can be unpinned in smaller chunks. 6166579324aSTerje Bergstrom */ 6176579324aSTerje Bergstrom void host1x_cdma_end(struct host1x_cdma *cdma, 6186579324aSTerje Bergstrom struct host1x_job *job) 6196579324aSTerje Bergstrom { 6206579324aSTerje Bergstrom struct host1x *host1x = cdma_to_host1x(cdma); 6216579324aSTerje Bergstrom bool idle = list_empty(&cdma->sync_queue); 6226579324aSTerje Bergstrom 6236579324aSTerje Bergstrom host1x_hw_cdma_flush(host1x, cdma); 6246579324aSTerje Bergstrom 6256579324aSTerje Bergstrom job->first_get = cdma->first_get; 6266579324aSTerje Bergstrom job->num_slots = cdma->slots_used; 6276579324aSTerje Bergstrom host1x_job_get(job); 6286579324aSTerje Bergstrom list_add_tail(&job->list, &cdma->sync_queue); 6296579324aSTerje Bergstrom 6306579324aSTerje Bergstrom /* start timer on idle -> active transitions */ 6316579324aSTerje Bergstrom if (job->timeout && idle) 6326579324aSTerje Bergstrom cdma_start_timer_locked(cdma, job); 6336579324aSTerje Bergstrom 6346579324aSTerje Bergstrom trace_host1x_cdma_end(dev_name(job->channel->dev)); 6356579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 6366579324aSTerje Bergstrom } 6376579324aSTerje Bergstrom 6386579324aSTerje Bergstrom /* 6396579324aSTerje Bergstrom * Update cdma state according to current sync point values 6406579324aSTerje Bergstrom */ 6416579324aSTerje Bergstrom void host1x_cdma_update(struct host1x_cdma *cdma) 6426579324aSTerje Bergstrom { 6436579324aSTerje Bergstrom mutex_lock(&cdma->lock); 6446579324aSTerje Bergstrom update_cdma_locked(cdma); 6456579324aSTerje Bergstrom mutex_unlock(&cdma->lock); 6466579324aSTerje Bergstrom } 647