1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Dave Airlie 30 */ 31 #include <linux/seq_file.h> 32 #include <asm/atomic.h> 33 #include <linux/wait.h> 34 #include <linux/list.h> 35 #include <linux/kref.h> 36 #include "drmP.h" 37 #include "drm.h" 38 #include "radeon_reg.h" 39 #include "radeon.h" 40 41 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) 42 { 43 unsigned long irq_flags; 44 45 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 46 if (fence->emited) { 47 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 48 return 0; 49 } 50 fence->seq = atomic_add_return(1, &rdev->fence_drv.seq); 51 if (!rdev->cp.ready) { 52 /* FIXME: cp is not running assume everythings is done right 53 * away 54 */ 55 WREG32(rdev->fence_drv.scratch_reg, fence->seq); 56 } else 57 radeon_fence_ring_emit(rdev, fence); 58 59 fence->emited = true; 60 fence->timeout = jiffies + ((2000 * HZ) / 1000); 61 list_del(&fence->list); 62 list_add_tail(&fence->list, &rdev->fence_drv.emited); 63 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 64 return 0; 65 } 66 67 static bool radeon_fence_poll_locked(struct radeon_device *rdev) 68 { 69 struct radeon_fence *fence; 70 struct list_head *i, *n; 71 uint32_t seq; 72 bool wake = false; 73 74 if (rdev == NULL) { 75 return true; 76 } 77 if (rdev->shutdown) { 78 return true; 79 } 80 seq = RREG32(rdev->fence_drv.scratch_reg); 81 rdev->fence_drv.last_seq = seq; 82 n = NULL; 83 list_for_each(i, &rdev->fence_drv.emited) { 84 fence = list_entry(i, struct radeon_fence, list); 85 if (fence->seq == seq) { 86 n = i; 87 break; 88 } 89 } 90 /* all fence previous to this one are considered as signaled */ 91 if (n) { 92 i = n; 93 do { 94 n = i->prev; 95 list_del(i); 96 list_add_tail(i, &rdev->fence_drv.signaled); 97 fence = list_entry(i, struct radeon_fence, list); 98 fence->signaled = true; 99 i = n; 100 } while (i != &rdev->fence_drv.emited); 101 wake = true; 102 } 103 return wake; 104 } 105 106 static void radeon_fence_destroy(struct kref *kref) 107 { 108 unsigned long irq_flags; 109 struct radeon_fence *fence; 110 111 fence = container_of(kref, struct radeon_fence, kref); 112 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); 113 list_del(&fence->list); 114 fence->emited = false; 115 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); 116 kfree(fence); 117 } 118 119 int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence) 120 { 121 unsigned long irq_flags; 122 123 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); 124 if ((*fence) == NULL) { 125 return -ENOMEM; 126 } 127 kref_init(&((*fence)->kref)); 128 (*fence)->rdev = rdev; 129 (*fence)->emited = false; 130 (*fence)->signaled = false; 131 (*fence)->seq = 0; 132 INIT_LIST_HEAD(&(*fence)->list); 133 134 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 135 list_add_tail(&(*fence)->list, &rdev->fence_drv.created); 136 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 137 return 0; 138 } 139 140 141 bool radeon_fence_signaled(struct radeon_fence *fence) 142 { 143 struct radeon_device *rdev = fence->rdev; 144 unsigned long irq_flags; 145 bool signaled = false; 146 147 if (rdev->gpu_lockup) { 148 return true; 149 } 150 if (fence == NULL) { 151 return true; 152 } 153 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); 154 signaled = fence->signaled; 155 /* if we are shuting down report all fence as signaled */ 156 if (fence->rdev->shutdown) { 157 signaled = true; 158 } 159 if (!fence->emited) { 160 WARN(1, "Querying an unemited fence : %p !\n", fence); 161 signaled = true; 162 } 163 if (!signaled) { 164 radeon_fence_poll_locked(fence->rdev); 165 signaled = fence->signaled; 166 } 167 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); 168 return signaled; 169 } 170 171 int r600_fence_wait(struct radeon_fence *fence, bool intr, bool lazy) 172 { 173 struct radeon_device *rdev; 174 int ret = 0; 175 176 rdev = fence->rdev; 177 178 __set_current_state(intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE); 179 180 while (1) { 181 if (radeon_fence_signaled(fence)) 182 break; 183 184 if (time_after_eq(jiffies, fence->timeout)) { 185 ret = -EBUSY; 186 break; 187 } 188 189 if (lazy) 190 schedule_timeout(1); 191 192 if (intr && signal_pending(current)) { 193 ret = -ERESTARTSYS; 194 break; 195 } 196 } 197 __set_current_state(TASK_RUNNING); 198 return ret; 199 } 200 201 202 int radeon_fence_wait(struct radeon_fence *fence, bool intr) 203 { 204 struct radeon_device *rdev; 205 unsigned long cur_jiffies; 206 unsigned long timeout; 207 bool expired = false; 208 int r; 209 210 if (fence == NULL) { 211 WARN(1, "Querying an invalid fence : %p !\n", fence); 212 return 0; 213 } 214 rdev = fence->rdev; 215 if (radeon_fence_signaled(fence)) { 216 return 0; 217 } 218 219 if (rdev->family >= CHIP_R600) { 220 r = r600_fence_wait(fence, intr, 0); 221 if (r == -ERESTARTSYS) 222 return -EBUSY; 223 return r; 224 } 225 226 retry: 227 cur_jiffies = jiffies; 228 timeout = HZ / 100; 229 if (time_after(fence->timeout, cur_jiffies)) { 230 timeout = fence->timeout - cur_jiffies; 231 } 232 233 if (intr) { 234 r = wait_event_interruptible_timeout(rdev->fence_drv.queue, 235 radeon_fence_signaled(fence), timeout); 236 if (unlikely(r == -ERESTARTSYS)) { 237 return -EBUSY; 238 } 239 } else { 240 r = wait_event_timeout(rdev->fence_drv.queue, 241 radeon_fence_signaled(fence), timeout); 242 } 243 if (unlikely(!radeon_fence_signaled(fence))) { 244 if (unlikely(r == 0)) { 245 expired = true; 246 } 247 if (unlikely(expired)) { 248 timeout = 1; 249 if (time_after(cur_jiffies, fence->timeout)) { 250 timeout = cur_jiffies - fence->timeout; 251 } 252 timeout = jiffies_to_msecs(timeout); 253 if (timeout > 500) { 254 DRM_ERROR("fence(%p:0x%08X) %lums timeout " 255 "going to reset GPU\n", 256 fence, fence->seq, timeout); 257 radeon_gpu_reset(rdev); 258 WREG32(rdev->fence_drv.scratch_reg, fence->seq); 259 } 260 } 261 goto retry; 262 } 263 if (unlikely(expired)) { 264 rdev->fence_drv.count_timeout++; 265 cur_jiffies = jiffies; 266 timeout = 1; 267 if (time_after(cur_jiffies, fence->timeout)) { 268 timeout = cur_jiffies - fence->timeout; 269 } 270 timeout = jiffies_to_msecs(timeout); 271 DRM_ERROR("fence(%p:0x%08X) %lums timeout\n", 272 fence, fence->seq, timeout); 273 DRM_ERROR("last signaled fence(0x%08X)\n", 274 rdev->fence_drv.last_seq); 275 } 276 return 0; 277 } 278 279 int radeon_fence_wait_next(struct radeon_device *rdev) 280 { 281 unsigned long irq_flags; 282 struct radeon_fence *fence; 283 int r; 284 285 if (rdev->gpu_lockup) { 286 return 0; 287 } 288 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 289 if (list_empty(&rdev->fence_drv.emited)) { 290 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 291 return 0; 292 } 293 fence = list_entry(rdev->fence_drv.emited.next, 294 struct radeon_fence, list); 295 radeon_fence_ref(fence); 296 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 297 r = radeon_fence_wait(fence, false); 298 radeon_fence_unref(&fence); 299 return r; 300 } 301 302 int radeon_fence_wait_last(struct radeon_device *rdev) 303 { 304 unsigned long irq_flags; 305 struct radeon_fence *fence; 306 int r; 307 308 if (rdev->gpu_lockup) { 309 return 0; 310 } 311 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 312 if (list_empty(&rdev->fence_drv.emited)) { 313 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 314 return 0; 315 } 316 fence = list_entry(rdev->fence_drv.emited.prev, 317 struct radeon_fence, list); 318 radeon_fence_ref(fence); 319 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 320 r = radeon_fence_wait(fence, false); 321 radeon_fence_unref(&fence); 322 return r; 323 } 324 325 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) 326 { 327 kref_get(&fence->kref); 328 return fence; 329 } 330 331 void radeon_fence_unref(struct radeon_fence **fence) 332 { 333 struct radeon_fence *tmp = *fence; 334 335 *fence = NULL; 336 if (tmp) { 337 kref_put(&tmp->kref, &radeon_fence_destroy); 338 } 339 } 340 341 void radeon_fence_process(struct radeon_device *rdev) 342 { 343 unsigned long irq_flags; 344 bool wake; 345 346 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 347 wake = radeon_fence_poll_locked(rdev); 348 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 349 if (wake) { 350 wake_up_all(&rdev->fence_drv.queue); 351 } 352 } 353 354 int radeon_fence_driver_init(struct radeon_device *rdev) 355 { 356 unsigned long irq_flags; 357 int r; 358 359 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 360 r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg); 361 if (r) { 362 DRM_ERROR("Fence failed to get a scratch register."); 363 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 364 return r; 365 } 366 WREG32(rdev->fence_drv.scratch_reg, 0); 367 atomic_set(&rdev->fence_drv.seq, 0); 368 INIT_LIST_HEAD(&rdev->fence_drv.created); 369 INIT_LIST_HEAD(&rdev->fence_drv.emited); 370 INIT_LIST_HEAD(&rdev->fence_drv.signaled); 371 rdev->fence_drv.count_timeout = 0; 372 init_waitqueue_head(&rdev->fence_drv.queue); 373 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 374 if (radeon_debugfs_fence_init(rdev)) { 375 DRM_ERROR("Failed to register debugfs file for fence !\n"); 376 } 377 return 0; 378 } 379 380 void radeon_fence_driver_fini(struct radeon_device *rdev) 381 { 382 unsigned long irq_flags; 383 384 wake_up_all(&rdev->fence_drv.queue); 385 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 386 radeon_scratch_free(rdev, rdev->fence_drv.scratch_reg); 387 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 388 DRM_INFO("radeon: fence finalized\n"); 389 } 390 391 392 /* 393 * Fence debugfs 394 */ 395 #if defined(CONFIG_DEBUG_FS) 396 static int radeon_debugfs_fence_info(struct seq_file *m, void *data) 397 { 398 struct drm_info_node *node = (struct drm_info_node *)m->private; 399 struct drm_device *dev = node->minor->dev; 400 struct radeon_device *rdev = dev->dev_private; 401 struct radeon_fence *fence; 402 403 seq_printf(m, "Last signaled fence 0x%08X\n", 404 RREG32(rdev->fence_drv.scratch_reg)); 405 if (!list_empty(&rdev->fence_drv.emited)) { 406 fence = list_entry(rdev->fence_drv.emited.prev, 407 struct radeon_fence, list); 408 seq_printf(m, "Last emited fence %p with 0x%08X\n", 409 fence, fence->seq); 410 } 411 return 0; 412 } 413 414 static struct drm_info_list radeon_debugfs_fence_list[] = { 415 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, 416 }; 417 #endif 418 419 int radeon_debugfs_fence_init(struct radeon_device *rdev) 420 { 421 #if defined(CONFIG_DEBUG_FS) 422 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1); 423 #else 424 return 0; 425 #endif 426 } 427