1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2016 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #include <linux/dma-fence.h> 8 9 #include "msm_drv.h" 10 #include "msm_fence.h" 11 #include "msm_gpu.h" 12 13 static struct msm_gpu *fctx2gpu(struct msm_fence_context *fctx) 14 { 15 struct msm_drm_private *priv = fctx->dev->dev_private; 16 return priv->gpu; 17 } 18 19 static enum hrtimer_restart deadline_timer(struct hrtimer *t) 20 { 21 struct msm_fence_context *fctx = container_of(t, 22 struct msm_fence_context, deadline_timer); 23 24 kthread_queue_work(fctx2gpu(fctx)->worker, &fctx->deadline_work); 25 26 return HRTIMER_NORESTART; 27 } 28 29 static void deadline_work(struct kthread_work *work) 30 { 31 struct msm_fence_context *fctx = container_of(work, 32 struct msm_fence_context, deadline_work); 33 34 /* If deadline fence has already passed, nothing to do: */ 35 if (msm_fence_completed(fctx, fctx->next_deadline_fence)) 36 return; 37 38 msm_devfreq_boost(fctx2gpu(fctx), 2); 39 } 40 41 42 struct msm_fence_context * 43 msm_fence_context_alloc(struct drm_device *dev, volatile uint32_t *fenceptr, 44 const char *name) 45 { 46 struct msm_fence_context *fctx; 47 static int index = 0; 48 49 fctx = kzalloc(sizeof(*fctx), GFP_KERNEL); 50 if (!fctx) 51 return ERR_PTR(-ENOMEM); 52 53 fctx->dev = dev; 54 strscpy(fctx->name, name, sizeof(fctx->name)); 55 fctx->context = dma_fence_context_alloc(1); 56 fctx->index = index++; 57 fctx->fenceptr = fenceptr; 58 spin_lock_init(&fctx->spinlock); 59 60 /* 61 * Start out close to the 32b fence rollover point, so we can 62 * catch bugs with fence comparisons. 63 */ 64 fctx->last_fence = 0xffffff00; 65 fctx->completed_fence = fctx->last_fence; 66 *fctx->fenceptr = fctx->last_fence; 67 68 hrtimer_init(&fctx->deadline_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 69 fctx->deadline_timer.function = deadline_timer; 70 71 kthread_init_work(&fctx->deadline_work, deadline_work); 72 73 fctx->next_deadline = ktime_get(); 74 75 return fctx; 76 } 77 78 void msm_fence_context_free(struct msm_fence_context *fctx) 79 { 80 kfree(fctx); 81 } 82 83 bool msm_fence_completed(struct msm_fence_context *fctx, uint32_t fence) 84 { 85 /* 86 * Note: Check completed_fence first, as fenceptr is in a write-combine 87 * mapping, so it will be more expensive to read. 88 */ 89 return (int32_t)(fctx->completed_fence - fence) >= 0 || 90 (int32_t)(*fctx->fenceptr - fence) >= 0; 91 } 92 93 /* called from irq handler and workqueue (in recover path) */ 94 void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence) 95 { 96 unsigned long flags; 97 98 spin_lock_irqsave(&fctx->spinlock, flags); 99 if (fence_after(fence, fctx->completed_fence)) 100 fctx->completed_fence = fence; 101 if (msm_fence_completed(fctx, fctx->next_deadline_fence)) 102 hrtimer_cancel(&fctx->deadline_timer); 103 spin_unlock_irqrestore(&fctx->spinlock, flags); 104 } 105 106 struct msm_fence { 107 struct dma_fence base; 108 struct msm_fence_context *fctx; 109 }; 110 111 static inline struct msm_fence *to_msm_fence(struct dma_fence *fence) 112 { 113 return container_of(fence, struct msm_fence, base); 114 } 115 116 static const char *msm_fence_get_driver_name(struct dma_fence *fence) 117 { 118 return "msm"; 119 } 120 121 static const char *msm_fence_get_timeline_name(struct dma_fence *fence) 122 { 123 struct msm_fence *f = to_msm_fence(fence); 124 return f->fctx->name; 125 } 126 127 static bool msm_fence_signaled(struct dma_fence *fence) 128 { 129 struct msm_fence *f = to_msm_fence(fence); 130 return msm_fence_completed(f->fctx, f->base.seqno); 131 } 132 133 static void msm_fence_set_deadline(struct dma_fence *fence, ktime_t deadline) 134 { 135 struct msm_fence *f = to_msm_fence(fence); 136 struct msm_fence_context *fctx = f->fctx; 137 unsigned long flags; 138 ktime_t now; 139 140 spin_lock_irqsave(&fctx->spinlock, flags); 141 now = ktime_get(); 142 143 if (ktime_after(now, fctx->next_deadline) || 144 ktime_before(deadline, fctx->next_deadline)) { 145 fctx->next_deadline = deadline; 146 fctx->next_deadline_fence = 147 max(fctx->next_deadline_fence, (uint32_t)fence->seqno); 148 149 /* 150 * Set timer to trigger boost 3ms before deadline, or 151 * if we are already less than 3ms before the deadline 152 * schedule boost work immediately. 153 */ 154 deadline = ktime_sub(deadline, ms_to_ktime(3)); 155 156 if (ktime_after(now, deadline)) { 157 kthread_queue_work(fctx2gpu(fctx)->worker, 158 &fctx->deadline_work); 159 } else { 160 hrtimer_start(&fctx->deadline_timer, deadline, 161 HRTIMER_MODE_ABS); 162 } 163 } 164 165 spin_unlock_irqrestore(&fctx->spinlock, flags); 166 } 167 168 static const struct dma_fence_ops msm_fence_ops = { 169 .get_driver_name = msm_fence_get_driver_name, 170 .get_timeline_name = msm_fence_get_timeline_name, 171 .signaled = msm_fence_signaled, 172 .set_deadline = msm_fence_set_deadline, 173 }; 174 175 struct dma_fence * 176 msm_fence_alloc(void) 177 { 178 struct msm_fence *f; 179 180 f = kzalloc(sizeof(*f), GFP_KERNEL); 181 if (!f) 182 return ERR_PTR(-ENOMEM); 183 184 return &f->base; 185 } 186 187 void 188 msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx) 189 { 190 struct msm_fence *f = to_msm_fence(fence); 191 192 f->fctx = fctx; 193 194 /* 195 * Until this point, the fence was just some pre-allocated memory, 196 * no-one should have taken a reference to it yet. 197 */ 198 WARN_ON(kref_read(&fence->refcount)); 199 200 dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock, 201 fctx->context, ++fctx->last_fence); 202 } 203