1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2017 The Linux Foundation. All rights reserved. 3 */ 4 5 #include "msm_gem.h" 6 #include "a5xx_gpu.h" 7 8 /* 9 * Try to transition the preemption state from old to new. Return 10 * true on success or false if the original state wasn't 'old' 11 */ 12 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu, 13 enum preempt_state old, enum preempt_state new) 14 { 15 enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state, 16 old, new); 17 18 return (cur == old); 19 } 20 21 /* 22 * Force the preemption state to the specified state. This is used in cases 23 * where the current state is known and won't change 24 */ 25 static inline void set_preempt_state(struct a5xx_gpu *gpu, 26 enum preempt_state new) 27 { 28 /* 29 * preempt_state may be read by other cores trying to trigger a 30 * preemption or in the interrupt handler so barriers are needed 31 * before... 32 */ 33 smp_mb__before_atomic(); 34 atomic_set(&gpu->preempt_state, new); 35 /* ... and after*/ 36 smp_mb__after_atomic(); 37 } 38 39 /* Write the most recent wptr for the given ring into the hardware */ 40 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 41 { 42 unsigned long flags; 43 uint32_t wptr; 44 45 if (!ring) 46 return; 47 48 spin_lock_irqsave(&ring->lock, flags); 49 wptr = get_wptr(ring); 50 spin_unlock_irqrestore(&ring->lock, flags); 51 52 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr); 53 } 54 55 /* Return the highest priority ringbuffer with something in it */ 56 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu) 57 { 58 unsigned long flags; 59 int i; 60 61 for (i = 0; i < gpu->nr_rings; i++) { 62 bool empty; 63 struct msm_ringbuffer *ring = gpu->rb[i]; 64 65 spin_lock_irqsave(&ring->lock, flags); 66 empty = (get_wptr(ring) == ring->memptrs->rptr); 67 spin_unlock_irqrestore(&ring->lock, flags); 68 69 if (!empty) 70 return ring; 71 } 72 73 return NULL; 74 } 75 76 static void a5xx_preempt_timer(struct timer_list *t) 77 { 78 struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer); 79 struct msm_gpu *gpu = &a5xx_gpu->base.base; 80 struct drm_device *dev = gpu->dev; 81 struct msm_drm_private *priv = dev->dev_private; 82 83 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED)) 84 return; 85 86 DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name); 87 queue_work(priv->wq, &gpu->recover_work); 88 } 89 90 /* Try to trigger a preemption switch */ 91 void a5xx_preempt_trigger(struct msm_gpu *gpu) 92 { 93 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 94 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 95 unsigned long flags; 96 struct msm_ringbuffer *ring; 97 98 if (gpu->nr_rings == 1) 99 return; 100 101 /* 102 * Try to start preemption by moving from NONE to START. If 103 * unsuccessful, a preemption is already in flight 104 */ 105 if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START)) 106 return; 107 108 /* Get the next ring to preempt to */ 109 ring = get_next_ring(gpu); 110 111 /* 112 * If no ring is populated or the highest priority ring is the current 113 * one do nothing except to update the wptr to the latest and greatest 114 */ 115 if (!ring || (a5xx_gpu->cur_ring == ring)) { 116 /* 117 * Its possible that while a preemption request is in progress 118 * from an irq context, a user context trying to submit might 119 * fail to update the write pointer, because it determines 120 * that the preempt state is not PREEMPT_NONE. 121 * 122 * Close the race by introducing an intermediate 123 * state PREEMPT_ABORT to let the submit path 124 * know that the ringbuffer is not going to change 125 * and can safely update the write pointer. 126 */ 127 128 set_preempt_state(a5xx_gpu, PREEMPT_ABORT); 129 update_wptr(gpu, a5xx_gpu->cur_ring); 130 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 131 return; 132 } 133 134 /* Make sure the wptr doesn't update while we're in motion */ 135 spin_lock_irqsave(&ring->lock, flags); 136 a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring); 137 spin_unlock_irqrestore(&ring->lock, flags); 138 139 /* Set the address of the incoming preemption record */ 140 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO, 141 REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI, 142 a5xx_gpu->preempt_iova[ring->id]); 143 144 a5xx_gpu->next_ring = ring; 145 146 /* Start a timer to catch a stuck preemption */ 147 mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000)); 148 149 /* Set the preemption state to triggered */ 150 set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED); 151 152 /* Make sure everything is written before hitting the button */ 153 wmb(); 154 155 /* And actually start the preemption */ 156 gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1); 157 } 158 159 void a5xx_preempt_irq(struct msm_gpu *gpu) 160 { 161 uint32_t status; 162 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 163 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 164 struct drm_device *dev = gpu->dev; 165 struct msm_drm_private *priv = dev->dev_private; 166 167 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING)) 168 return; 169 170 /* Delete the preemption watchdog timer */ 171 del_timer(&a5xx_gpu->preempt_timer); 172 173 /* 174 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before 175 * firing the interrupt, but there is a non zero chance of a hardware 176 * condition or a software race that could set it again before we have a 177 * chance to finish. If that happens, log and go for recovery 178 */ 179 status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL); 180 if (unlikely(status)) { 181 set_preempt_state(a5xx_gpu, PREEMPT_FAULTED); 182 DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n", 183 gpu->name); 184 queue_work(priv->wq, &gpu->recover_work); 185 return; 186 } 187 188 a5xx_gpu->cur_ring = a5xx_gpu->next_ring; 189 a5xx_gpu->next_ring = NULL; 190 191 update_wptr(gpu, a5xx_gpu->cur_ring); 192 193 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 194 } 195 196 void a5xx_preempt_hw_init(struct msm_gpu *gpu) 197 { 198 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 199 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 200 int i; 201 202 /* Always come up on rb 0 */ 203 a5xx_gpu->cur_ring = gpu->rb[0]; 204 205 /* No preemption if we only have one ring */ 206 if (gpu->nr_rings == 1) 207 return; 208 209 for (i = 0; i < gpu->nr_rings; i++) { 210 a5xx_gpu->preempt[i]->wptr = 0; 211 a5xx_gpu->preempt[i]->rptr = 0; 212 a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova; 213 } 214 215 /* Write a 0 to signal that we aren't switching pagetables */ 216 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 217 REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0); 218 219 /* Reset the preemption state */ 220 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 221 } 222 223 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu, 224 struct msm_ringbuffer *ring) 225 { 226 struct adreno_gpu *adreno_gpu = &a5xx_gpu->base; 227 struct msm_gpu *gpu = &adreno_gpu->base; 228 struct a5xx_preempt_record *ptr; 229 struct drm_gem_object *bo = NULL; 230 u64 iova = 0; 231 232 ptr = msm_gem_kernel_new(gpu->dev, 233 A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE, 234 MSM_BO_UNCACHED, gpu->aspace, &bo, &iova); 235 236 if (IS_ERR(ptr)) 237 return PTR_ERR(ptr); 238 239 msm_gem_object_set_name(bo, "preempt"); 240 241 a5xx_gpu->preempt_bo[ring->id] = bo; 242 a5xx_gpu->preempt_iova[ring->id] = iova; 243 a5xx_gpu->preempt[ring->id] = ptr; 244 245 /* Set up the defaults on the preemption record */ 246 247 ptr->magic = A5XX_PREEMPT_RECORD_MAGIC; 248 ptr->info = 0; 249 ptr->data = 0; 250 ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT; 251 ptr->rptr_addr = rbmemptr(ring, rptr); 252 ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE; 253 254 return 0; 255 } 256 257 void a5xx_preempt_fini(struct msm_gpu *gpu) 258 { 259 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 260 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 261 int i; 262 263 for (i = 0; i < gpu->nr_rings; i++) 264 msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace, true); 265 } 266 267 void a5xx_preempt_init(struct msm_gpu *gpu) 268 { 269 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 270 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 271 int i; 272 273 /* No preemption if we only have one ring */ 274 if (gpu->nr_rings <= 1) 275 return; 276 277 for (i = 0; i < gpu->nr_rings; i++) { 278 if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) { 279 /* 280 * On any failure our adventure is over. Clean up and 281 * set nr_rings to 1 to force preemption off 282 */ 283 a5xx_preempt_fini(gpu); 284 gpu->nr_rings = 1; 285 286 return; 287 } 288 } 289 290 timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0); 291 } 292