1 /* Copyright (c) 2017 The Linux Foundation. All rights reserved. 2 * 3 * This program is free software; you can redistribute it and/or modify 4 * it under the terms of the GNU General Public License version 2 and 5 * only version 2 as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 */ 13 14 #include "msm_gem.h" 15 #include "a5xx_gpu.h" 16 17 /* 18 * Try to transition the preemption state from old to new. Return 19 * true on success or false if the original state wasn't 'old' 20 */ 21 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu, 22 enum preempt_state old, enum preempt_state new) 23 { 24 enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state, 25 old, new); 26 27 return (cur == old); 28 } 29 30 /* 31 * Force the preemption state to the specified state. This is used in cases 32 * where the current state is known and won't change 33 */ 34 static inline void set_preempt_state(struct a5xx_gpu *gpu, 35 enum preempt_state new) 36 { 37 /* 38 * preempt_state may be read by other cores trying to trigger a 39 * preemption or in the interrupt handler so barriers are needed 40 * before... 41 */ 42 smp_mb__before_atomic(); 43 atomic_set(&gpu->preempt_state, new); 44 /* ... and after*/ 45 smp_mb__after_atomic(); 46 } 47 48 /* Write the most recent wptr for the given ring into the hardware */ 49 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) 50 { 51 unsigned long flags; 52 uint32_t wptr; 53 54 if (!ring) 55 return; 56 57 spin_lock_irqsave(&ring->lock, flags); 58 wptr = get_wptr(ring); 59 spin_unlock_irqrestore(&ring->lock, flags); 60 61 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr); 62 } 63 64 /* Return the highest priority ringbuffer with something in it */ 65 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu) 66 { 67 unsigned long flags; 68 int i; 69 70 for (i = 0; i < gpu->nr_rings; i++) { 71 bool empty; 72 struct msm_ringbuffer *ring = gpu->rb[i]; 73 74 spin_lock_irqsave(&ring->lock, flags); 75 empty = (get_wptr(ring) == ring->memptrs->rptr); 76 spin_unlock_irqrestore(&ring->lock, flags); 77 78 if (!empty) 79 return ring; 80 } 81 82 return NULL; 83 } 84 85 static void a5xx_preempt_timer(struct timer_list *t) 86 { 87 struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer); 88 struct msm_gpu *gpu = &a5xx_gpu->base.base; 89 struct drm_device *dev = gpu->dev; 90 struct msm_drm_private *priv = dev->dev_private; 91 92 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED)) 93 return; 94 95 dev_err(dev->dev, "%s: preemption timed out\n", gpu->name); 96 queue_work(priv->wq, &gpu->recover_work); 97 } 98 99 /* Try to trigger a preemption switch */ 100 void a5xx_preempt_trigger(struct msm_gpu *gpu) 101 { 102 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 103 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 104 unsigned long flags; 105 struct msm_ringbuffer *ring; 106 107 if (gpu->nr_rings == 1) 108 return; 109 110 /* 111 * Try to start preemption by moving from NONE to START. If 112 * unsuccessful, a preemption is already in flight 113 */ 114 if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START)) 115 return; 116 117 /* Get the next ring to preempt to */ 118 ring = get_next_ring(gpu); 119 120 /* 121 * If no ring is populated or the highest priority ring is the current 122 * one do nothing except to update the wptr to the latest and greatest 123 */ 124 if (!ring || (a5xx_gpu->cur_ring == ring)) { 125 /* 126 * Its possible that while a preemption request is in progress 127 * from an irq context, a user context trying to submit might 128 * fail to update the write pointer, because it determines 129 * that the preempt state is not PREEMPT_NONE. 130 * 131 * Close the race by introducing an intermediate 132 * state PREEMPT_ABORT to let the submit path 133 * know that the ringbuffer is not going to change 134 * and can safely update the write pointer. 135 */ 136 137 set_preempt_state(a5xx_gpu, PREEMPT_ABORT); 138 update_wptr(gpu, a5xx_gpu->cur_ring); 139 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 140 return; 141 } 142 143 /* Make sure the wptr doesn't update while we're in motion */ 144 spin_lock_irqsave(&ring->lock, flags); 145 a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring); 146 spin_unlock_irqrestore(&ring->lock, flags); 147 148 /* Set the address of the incoming preemption record */ 149 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO, 150 REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI, 151 a5xx_gpu->preempt_iova[ring->id]); 152 153 a5xx_gpu->next_ring = ring; 154 155 /* Start a timer to catch a stuck preemption */ 156 mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000)); 157 158 /* Set the preemption state to triggered */ 159 set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED); 160 161 /* Make sure everything is written before hitting the button */ 162 wmb(); 163 164 /* And actually start the preemption */ 165 gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1); 166 } 167 168 void a5xx_preempt_irq(struct msm_gpu *gpu) 169 { 170 uint32_t status; 171 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 172 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 173 struct drm_device *dev = gpu->dev; 174 struct msm_drm_private *priv = dev->dev_private; 175 176 if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING)) 177 return; 178 179 /* Delete the preemption watchdog timer */ 180 del_timer(&a5xx_gpu->preempt_timer); 181 182 /* 183 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before 184 * firing the interrupt, but there is a non zero chance of a hardware 185 * condition or a software race that could set it again before we have a 186 * chance to finish. If that happens, log and go for recovery 187 */ 188 status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL); 189 if (unlikely(status)) { 190 set_preempt_state(a5xx_gpu, PREEMPT_FAULTED); 191 dev_err(dev->dev, "%s: Preemption failed to complete\n", 192 gpu->name); 193 queue_work(priv->wq, &gpu->recover_work); 194 return; 195 } 196 197 a5xx_gpu->cur_ring = a5xx_gpu->next_ring; 198 a5xx_gpu->next_ring = NULL; 199 200 update_wptr(gpu, a5xx_gpu->cur_ring); 201 202 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 203 } 204 205 void a5xx_preempt_hw_init(struct msm_gpu *gpu) 206 { 207 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 208 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 209 int i; 210 211 /* Always come up on rb 0 */ 212 a5xx_gpu->cur_ring = gpu->rb[0]; 213 214 /* No preemption if we only have one ring */ 215 if (gpu->nr_rings == 1) 216 return; 217 218 for (i = 0; i < gpu->nr_rings; i++) { 219 a5xx_gpu->preempt[i]->wptr = 0; 220 a5xx_gpu->preempt[i]->rptr = 0; 221 a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova; 222 } 223 224 /* Write a 0 to signal that we aren't switching pagetables */ 225 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 226 REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0); 227 228 /* Reset the preemption state */ 229 set_preempt_state(a5xx_gpu, PREEMPT_NONE); 230 } 231 232 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu, 233 struct msm_ringbuffer *ring) 234 { 235 struct adreno_gpu *adreno_gpu = &a5xx_gpu->base; 236 struct msm_gpu *gpu = &adreno_gpu->base; 237 struct a5xx_preempt_record *ptr; 238 struct drm_gem_object *bo = NULL; 239 u64 iova = 0; 240 241 ptr = msm_gem_kernel_new(gpu->dev, 242 A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE, 243 MSM_BO_UNCACHED, gpu->aspace, &bo, &iova); 244 245 if (IS_ERR(ptr)) 246 return PTR_ERR(ptr); 247 248 a5xx_gpu->preempt_bo[ring->id] = bo; 249 a5xx_gpu->preempt_iova[ring->id] = iova; 250 a5xx_gpu->preempt[ring->id] = ptr; 251 252 /* Set up the defaults on the preemption record */ 253 254 ptr->magic = A5XX_PREEMPT_RECORD_MAGIC; 255 ptr->info = 0; 256 ptr->data = 0; 257 ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT; 258 ptr->rptr_addr = rbmemptr(ring, rptr); 259 ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE; 260 261 return 0; 262 } 263 264 void a5xx_preempt_fini(struct msm_gpu *gpu) 265 { 266 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 267 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 268 int i; 269 270 for (i = 0; i < gpu->nr_rings; i++) { 271 if (!a5xx_gpu->preempt_bo[i]) 272 continue; 273 274 msm_gem_put_vaddr(a5xx_gpu->preempt_bo[i]); 275 276 if (a5xx_gpu->preempt_iova[i]) 277 msm_gem_put_iova(a5xx_gpu->preempt_bo[i], gpu->aspace); 278 279 drm_gem_object_put(a5xx_gpu->preempt_bo[i]); 280 a5xx_gpu->preempt_bo[i] = NULL; 281 } 282 } 283 284 void a5xx_preempt_init(struct msm_gpu *gpu) 285 { 286 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); 287 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); 288 int i; 289 290 /* No preemption if we only have one ring */ 291 if (gpu->nr_rings <= 1) 292 return; 293 294 for (i = 0; i < gpu->nr_rings; i++) { 295 if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) { 296 /* 297 * On any failure our adventure is over. Clean up and 298 * set nr_rings to 1 to force preemption off 299 */ 300 a5xx_preempt_fini(gpu); 301 gpu->nr_rings = 1; 302 303 return; 304 } 305 } 306 307 timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0); 308 } 309