1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Dave Airlie 30 */ 31 #include <linux/seq_file.h> 32 #include <asm/atomic.h> 33 #include <linux/wait.h> 34 #include <linux/list.h> 35 #include <linux/kref.h> 36 #include <linux/slab.h> 37 #include "drmP.h" 38 #include "drm.h" 39 #include "radeon_reg.h" 40 #include "radeon.h" 41 #include "radeon_trace.h" 42 43 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) 44 { 45 unsigned long irq_flags; 46 47 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 48 if (fence->emited) { 49 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 50 return 0; 51 } 52 fence->seq = atomic_add_return(1, &rdev->fence_drv.seq); 53 if (!rdev->cp.ready) { 54 /* FIXME: cp is not running assume everythings is done right 55 * away 56 */ 57 WREG32(rdev->fence_drv.scratch_reg, fence->seq); 58 } else 59 radeon_fence_ring_emit(rdev, fence); 60 61 trace_radeon_fence_emit(rdev->ddev, fence->seq); 62 fence->emited = true; 63 list_del(&fence->list); 64 list_add_tail(&fence->list, &rdev->fence_drv.emited); 65 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 66 return 0; 67 } 68 69 static bool radeon_fence_poll_locked(struct radeon_device *rdev) 70 { 71 struct radeon_fence *fence; 72 struct list_head *i, *n; 73 uint32_t seq; 74 bool wake = false; 75 unsigned long cjiffies; 76 77 if (rdev->wb.enabled) { 78 u32 scratch_index; 79 if (rdev->wb.use_event) 80 scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; 81 else 82 scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; 83 seq = rdev->wb.wb[scratch_index/4]; 84 } else 85 seq = RREG32(rdev->fence_drv.scratch_reg); 86 if (seq != rdev->fence_drv.last_seq) { 87 rdev->fence_drv.last_seq = seq; 88 rdev->fence_drv.last_jiffies = jiffies; 89 rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT; 90 } else { 91 cjiffies = jiffies; 92 if (time_after(cjiffies, rdev->fence_drv.last_jiffies)) { 93 cjiffies -= rdev->fence_drv.last_jiffies; 94 if (time_after(rdev->fence_drv.last_timeout, cjiffies)) { 95 /* update the timeout */ 96 rdev->fence_drv.last_timeout -= cjiffies; 97 } else { 98 /* the 500ms timeout is elapsed we should test 99 * for GPU lockup 100 */ 101 rdev->fence_drv.last_timeout = 1; 102 } 103 } else { 104 /* wrap around update last jiffies, we will just wait 105 * a little longer 106 */ 107 rdev->fence_drv.last_jiffies = cjiffies; 108 } 109 return false; 110 } 111 n = NULL; 112 list_for_each(i, &rdev->fence_drv.emited) { 113 fence = list_entry(i, struct radeon_fence, list); 114 if (fence->seq == seq) { 115 n = i; 116 break; 117 } 118 } 119 /* all fence previous to this one are considered as signaled */ 120 if (n) { 121 i = n; 122 do { 123 n = i->prev; 124 list_del(i); 125 list_add_tail(i, &rdev->fence_drv.signaled); 126 fence = list_entry(i, struct radeon_fence, list); 127 fence->signaled = true; 128 i = n; 129 } while (i != &rdev->fence_drv.emited); 130 wake = true; 131 } 132 return wake; 133 } 134 135 static void radeon_fence_destroy(struct kref *kref) 136 { 137 unsigned long irq_flags; 138 struct radeon_fence *fence; 139 140 fence = container_of(kref, struct radeon_fence, kref); 141 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); 142 list_del(&fence->list); 143 fence->emited = false; 144 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); 145 kfree(fence); 146 } 147 148 int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence) 149 { 150 unsigned long irq_flags; 151 152 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); 153 if ((*fence) == NULL) { 154 return -ENOMEM; 155 } 156 kref_init(&((*fence)->kref)); 157 (*fence)->rdev = rdev; 158 (*fence)->emited = false; 159 (*fence)->signaled = false; 160 (*fence)->seq = 0; 161 INIT_LIST_HEAD(&(*fence)->list); 162 163 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 164 list_add_tail(&(*fence)->list, &rdev->fence_drv.created); 165 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 166 return 0; 167 } 168 169 170 bool radeon_fence_signaled(struct radeon_fence *fence) 171 { 172 unsigned long irq_flags; 173 bool signaled = false; 174 175 if (!fence) 176 return true; 177 178 if (fence->rdev->gpu_lockup) 179 return true; 180 181 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); 182 signaled = fence->signaled; 183 /* if we are shuting down report all fence as signaled */ 184 if (fence->rdev->shutdown) { 185 signaled = true; 186 } 187 if (!fence->emited) { 188 WARN(1, "Querying an unemited fence : %p !\n", fence); 189 signaled = true; 190 } 191 if (!signaled) { 192 radeon_fence_poll_locked(fence->rdev); 193 signaled = fence->signaled; 194 } 195 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); 196 return signaled; 197 } 198 199 int radeon_fence_wait(struct radeon_fence *fence, bool intr) 200 { 201 struct radeon_device *rdev; 202 unsigned long irq_flags, timeout; 203 u32 seq; 204 int r; 205 206 if (fence == NULL) { 207 WARN(1, "Querying an invalid fence : %p !\n", fence); 208 return 0; 209 } 210 rdev = fence->rdev; 211 if (radeon_fence_signaled(fence)) { 212 return 0; 213 } 214 timeout = rdev->fence_drv.last_timeout; 215 retry: 216 /* save current sequence used to check for GPU lockup */ 217 seq = rdev->fence_drv.last_seq; 218 trace_radeon_fence_wait_begin(rdev->ddev, seq); 219 if (intr) { 220 radeon_irq_kms_sw_irq_get(rdev); 221 r = wait_event_interruptible_timeout(rdev->fence_drv.queue, 222 radeon_fence_signaled(fence), timeout); 223 radeon_irq_kms_sw_irq_put(rdev); 224 if (unlikely(r < 0)) { 225 return r; 226 } 227 } else { 228 radeon_irq_kms_sw_irq_get(rdev); 229 r = wait_event_timeout(rdev->fence_drv.queue, 230 radeon_fence_signaled(fence), timeout); 231 radeon_irq_kms_sw_irq_put(rdev); 232 } 233 trace_radeon_fence_wait_end(rdev->ddev, seq); 234 if (unlikely(!radeon_fence_signaled(fence))) { 235 /* we were interrupted for some reason and fence isn't 236 * isn't signaled yet, resume wait 237 */ 238 if (r) { 239 timeout = r; 240 goto retry; 241 } 242 /* don't protect read access to rdev->fence_drv.last_seq 243 * if we experiencing a lockup the value doesn't change 244 */ 245 if (seq == rdev->fence_drv.last_seq && radeon_gpu_is_lockup(rdev)) { 246 /* good news we believe it's a lockup */ 247 WARN(1, "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n", 248 fence->seq, seq); 249 /* FIXME: what should we do ? marking everyone 250 * as signaled for now 251 */ 252 rdev->gpu_lockup = true; 253 r = radeon_gpu_reset(rdev); 254 if (r) 255 return r; 256 WREG32(rdev->fence_drv.scratch_reg, fence->seq); 257 rdev->gpu_lockup = false; 258 } 259 timeout = RADEON_FENCE_JIFFIES_TIMEOUT; 260 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 261 rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT; 262 rdev->fence_drv.last_jiffies = jiffies; 263 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 264 goto retry; 265 } 266 return 0; 267 } 268 269 int radeon_fence_wait_next(struct radeon_device *rdev) 270 { 271 unsigned long irq_flags; 272 struct radeon_fence *fence; 273 int r; 274 275 if (rdev->gpu_lockup) { 276 return 0; 277 } 278 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 279 if (list_empty(&rdev->fence_drv.emited)) { 280 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 281 return 0; 282 } 283 fence = list_entry(rdev->fence_drv.emited.next, 284 struct radeon_fence, list); 285 radeon_fence_ref(fence); 286 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 287 r = radeon_fence_wait(fence, false); 288 radeon_fence_unref(&fence); 289 return r; 290 } 291 292 int radeon_fence_wait_last(struct radeon_device *rdev) 293 { 294 unsigned long irq_flags; 295 struct radeon_fence *fence; 296 int r; 297 298 if (rdev->gpu_lockup) { 299 return 0; 300 } 301 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 302 if (list_empty(&rdev->fence_drv.emited)) { 303 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 304 return 0; 305 } 306 fence = list_entry(rdev->fence_drv.emited.prev, 307 struct radeon_fence, list); 308 radeon_fence_ref(fence); 309 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 310 r = radeon_fence_wait(fence, false); 311 radeon_fence_unref(&fence); 312 return r; 313 } 314 315 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) 316 { 317 kref_get(&fence->kref); 318 return fence; 319 } 320 321 void radeon_fence_unref(struct radeon_fence **fence) 322 { 323 struct radeon_fence *tmp = *fence; 324 325 *fence = NULL; 326 if (tmp) { 327 kref_put(&tmp->kref, &radeon_fence_destroy); 328 } 329 } 330 331 void radeon_fence_process(struct radeon_device *rdev) 332 { 333 unsigned long irq_flags; 334 bool wake; 335 336 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 337 wake = radeon_fence_poll_locked(rdev); 338 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 339 if (wake) { 340 wake_up_all(&rdev->fence_drv.queue); 341 } 342 } 343 344 int radeon_fence_driver_init(struct radeon_device *rdev) 345 { 346 unsigned long irq_flags; 347 int r; 348 349 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 350 r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg); 351 if (r) { 352 dev_err(rdev->dev, "fence failed to get scratch register\n"); 353 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 354 return r; 355 } 356 WREG32(rdev->fence_drv.scratch_reg, 0); 357 atomic_set(&rdev->fence_drv.seq, 0); 358 INIT_LIST_HEAD(&rdev->fence_drv.created); 359 INIT_LIST_HEAD(&rdev->fence_drv.emited); 360 INIT_LIST_HEAD(&rdev->fence_drv.signaled); 361 init_waitqueue_head(&rdev->fence_drv.queue); 362 rdev->fence_drv.initialized = true; 363 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 364 if (radeon_debugfs_fence_init(rdev)) { 365 dev_err(rdev->dev, "fence debugfs file creation failed\n"); 366 } 367 return 0; 368 } 369 370 void radeon_fence_driver_fini(struct radeon_device *rdev) 371 { 372 unsigned long irq_flags; 373 374 if (!rdev->fence_drv.initialized) 375 return; 376 wake_up_all(&rdev->fence_drv.queue); 377 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 378 radeon_scratch_free(rdev, rdev->fence_drv.scratch_reg); 379 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 380 rdev->fence_drv.initialized = false; 381 } 382 383 384 /* 385 * Fence debugfs 386 */ 387 #if defined(CONFIG_DEBUG_FS) 388 static int radeon_debugfs_fence_info(struct seq_file *m, void *data) 389 { 390 struct drm_info_node *node = (struct drm_info_node *)m->private; 391 struct drm_device *dev = node->minor->dev; 392 struct radeon_device *rdev = dev->dev_private; 393 struct radeon_fence *fence; 394 395 seq_printf(m, "Last signaled fence 0x%08X\n", 396 RREG32(rdev->fence_drv.scratch_reg)); 397 if (!list_empty(&rdev->fence_drv.emited)) { 398 fence = list_entry(rdev->fence_drv.emited.prev, 399 struct radeon_fence, list); 400 seq_printf(m, "Last emited fence %p with 0x%08X\n", 401 fence, fence->seq); 402 } 403 return 0; 404 } 405 406 static struct drm_info_list radeon_debugfs_fence_list[] = { 407 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, 408 }; 409 #endif 410 411 int radeon_debugfs_fence_init(struct radeon_device *rdev) 412 { 413 #if defined(CONFIG_DEBUG_FS) 414 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1); 415 #else 416 return 0; 417 #endif 418 } 419