1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Dave Airlie 30 */ 31 #include <linux/seq_file.h> 32 #include <asm/atomic.h> 33 #include <linux/wait.h> 34 #include <linux/list.h> 35 #include <linux/kref.h> 36 #include "drmP.h" 37 #include "drm.h" 38 #include "radeon_reg.h" 39 #include "radeon.h" 40 41 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) 42 { 43 unsigned long irq_flags; 44 45 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 46 if (fence->emited) { 47 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 48 return 0; 49 } 50 fence->seq = atomic_add_return(1, &rdev->fence_drv.seq); 51 if (!rdev->cp.ready) { 52 /* FIXME: cp is not running assume everythings is done right 53 * away 54 */ 55 WREG32(rdev->fence_drv.scratch_reg, fence->seq); 56 } else { 57 radeon_fence_ring_emit(rdev, fence); 58 } 59 fence->emited = true; 60 fence->timeout = jiffies + ((2000 * HZ) / 1000); 61 list_del(&fence->list); 62 list_add_tail(&fence->list, &rdev->fence_drv.emited); 63 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 64 return 0; 65 } 66 67 static bool radeon_fence_poll_locked(struct radeon_device *rdev) 68 { 69 struct radeon_fence *fence; 70 struct list_head *i, *n; 71 uint32_t seq; 72 bool wake = false; 73 74 if (rdev == NULL) { 75 return true; 76 } 77 if (rdev->shutdown) { 78 return true; 79 } 80 seq = RREG32(rdev->fence_drv.scratch_reg); 81 rdev->fence_drv.last_seq = seq; 82 n = NULL; 83 list_for_each(i, &rdev->fence_drv.emited) { 84 fence = list_entry(i, struct radeon_fence, list); 85 if (fence->seq == seq) { 86 n = i; 87 break; 88 } 89 } 90 /* all fence previous to this one are considered as signaled */ 91 if (n) { 92 i = n; 93 do { 94 n = i->prev; 95 list_del(i); 96 list_add_tail(i, &rdev->fence_drv.signaled); 97 fence = list_entry(i, struct radeon_fence, list); 98 fence->signaled = true; 99 i = n; 100 } while (i != &rdev->fence_drv.emited); 101 wake = true; 102 } 103 return wake; 104 } 105 106 static void radeon_fence_destroy(struct kref *kref) 107 { 108 unsigned long irq_flags; 109 struct radeon_fence *fence; 110 111 fence = container_of(kref, struct radeon_fence, kref); 112 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); 113 list_del(&fence->list); 114 fence->emited = false; 115 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); 116 kfree(fence); 117 } 118 119 int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence) 120 { 121 unsigned long irq_flags; 122 123 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); 124 if ((*fence) == NULL) { 125 return -ENOMEM; 126 } 127 kref_init(&((*fence)->kref)); 128 (*fence)->rdev = rdev; 129 (*fence)->emited = false; 130 (*fence)->signaled = false; 131 (*fence)->seq = 0; 132 INIT_LIST_HEAD(&(*fence)->list); 133 134 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 135 list_add_tail(&(*fence)->list, &rdev->fence_drv.created); 136 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 137 return 0; 138 } 139 140 141 bool radeon_fence_signaled(struct radeon_fence *fence) 142 { 143 struct radeon_device *rdev = fence->rdev; 144 unsigned long irq_flags; 145 bool signaled = false; 146 147 if (rdev->gpu_lockup) { 148 return true; 149 } 150 if (fence == NULL) { 151 return true; 152 } 153 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); 154 signaled = fence->signaled; 155 /* if we are shuting down report all fence as signaled */ 156 if (fence->rdev->shutdown) { 157 signaled = true; 158 } 159 if (!fence->emited) { 160 WARN(1, "Querying an unemited fence : %p !\n", fence); 161 signaled = true; 162 } 163 if (!signaled) { 164 radeon_fence_poll_locked(fence->rdev); 165 signaled = fence->signaled; 166 } 167 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); 168 return signaled; 169 } 170 171 int radeon_fence_wait(struct radeon_fence *fence, bool interruptible) 172 { 173 struct radeon_device *rdev; 174 unsigned long cur_jiffies; 175 unsigned long timeout; 176 bool expired = false; 177 int r; 178 179 180 if (fence == NULL) { 181 WARN(1, "Querying an invalid fence : %p !\n", fence); 182 return 0; 183 } 184 rdev = fence->rdev; 185 if (radeon_fence_signaled(fence)) { 186 return 0; 187 } 188 retry: 189 cur_jiffies = jiffies; 190 timeout = HZ / 100; 191 if (time_after(fence->timeout, cur_jiffies)) { 192 timeout = fence->timeout - cur_jiffies; 193 } 194 if (interruptible) { 195 r = wait_event_interruptible_timeout(rdev->fence_drv.queue, 196 radeon_fence_signaled(fence), timeout); 197 if (unlikely(r == -ERESTARTSYS)) { 198 return -EBUSY; 199 } 200 } else { 201 r = wait_event_timeout(rdev->fence_drv.queue, 202 radeon_fence_signaled(fence), timeout); 203 } 204 if (unlikely(!radeon_fence_signaled(fence))) { 205 if (unlikely(r == 0)) { 206 expired = true; 207 } 208 if (unlikely(expired)) { 209 timeout = 1; 210 if (time_after(cur_jiffies, fence->timeout)) { 211 timeout = cur_jiffies - fence->timeout; 212 } 213 timeout = jiffies_to_msecs(timeout); 214 if (timeout > 500) { 215 DRM_ERROR("fence(%p:0x%08X) %lums timeout " 216 "going to reset GPU\n", 217 fence, fence->seq, timeout); 218 radeon_gpu_reset(rdev); 219 WREG32(rdev->fence_drv.scratch_reg, fence->seq); 220 } 221 } 222 goto retry; 223 } 224 if (unlikely(expired)) { 225 rdev->fence_drv.count_timeout++; 226 cur_jiffies = jiffies; 227 timeout = 1; 228 if (time_after(cur_jiffies, fence->timeout)) { 229 timeout = cur_jiffies - fence->timeout; 230 } 231 timeout = jiffies_to_msecs(timeout); 232 DRM_ERROR("fence(%p:0x%08X) %lums timeout\n", 233 fence, fence->seq, timeout); 234 DRM_ERROR("last signaled fence(0x%08X)\n", 235 rdev->fence_drv.last_seq); 236 } 237 return 0; 238 } 239 240 int radeon_fence_wait_next(struct radeon_device *rdev) 241 { 242 unsigned long irq_flags; 243 struct radeon_fence *fence; 244 int r; 245 246 if (rdev->gpu_lockup) { 247 return 0; 248 } 249 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 250 if (list_empty(&rdev->fence_drv.emited)) { 251 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 252 return 0; 253 } 254 fence = list_entry(rdev->fence_drv.emited.next, 255 struct radeon_fence, list); 256 radeon_fence_ref(fence); 257 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 258 r = radeon_fence_wait(fence, false); 259 radeon_fence_unref(&fence); 260 return r; 261 } 262 263 int radeon_fence_wait_last(struct radeon_device *rdev) 264 { 265 unsigned long irq_flags; 266 struct radeon_fence *fence; 267 int r; 268 269 if (rdev->gpu_lockup) { 270 return 0; 271 } 272 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 273 if (list_empty(&rdev->fence_drv.emited)) { 274 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 275 return 0; 276 } 277 fence = list_entry(rdev->fence_drv.emited.prev, 278 struct radeon_fence, list); 279 radeon_fence_ref(fence); 280 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 281 r = radeon_fence_wait(fence, false); 282 radeon_fence_unref(&fence); 283 return r; 284 } 285 286 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) 287 { 288 kref_get(&fence->kref); 289 return fence; 290 } 291 292 void radeon_fence_unref(struct radeon_fence **fence) 293 { 294 struct radeon_fence *tmp = *fence; 295 296 *fence = NULL; 297 if (tmp) { 298 kref_put(&tmp->kref, &radeon_fence_destroy); 299 } 300 } 301 302 void radeon_fence_process(struct radeon_device *rdev) 303 { 304 unsigned long irq_flags; 305 bool wake; 306 307 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 308 wake = radeon_fence_poll_locked(rdev); 309 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 310 if (wake) { 311 wake_up_all(&rdev->fence_drv.queue); 312 } 313 } 314 315 int radeon_fence_driver_init(struct radeon_device *rdev) 316 { 317 unsigned long irq_flags; 318 int r; 319 320 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 321 r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg); 322 if (r) { 323 DRM_ERROR("Fence failed to get a scratch register."); 324 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 325 return r; 326 } 327 WREG32(rdev->fence_drv.scratch_reg, 0); 328 atomic_set(&rdev->fence_drv.seq, 0); 329 INIT_LIST_HEAD(&rdev->fence_drv.created); 330 INIT_LIST_HEAD(&rdev->fence_drv.emited); 331 INIT_LIST_HEAD(&rdev->fence_drv.signaled); 332 rdev->fence_drv.count_timeout = 0; 333 init_waitqueue_head(&rdev->fence_drv.queue); 334 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 335 if (radeon_debugfs_fence_init(rdev)) { 336 DRM_ERROR("Failed to register debugfs file for fence !\n"); 337 } 338 return 0; 339 } 340 341 void radeon_fence_driver_fini(struct radeon_device *rdev) 342 { 343 unsigned long irq_flags; 344 345 wake_up_all(&rdev->fence_drv.queue); 346 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 347 radeon_scratch_free(rdev, rdev->fence_drv.scratch_reg); 348 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 349 DRM_INFO("radeon: fence finalized\n"); 350 } 351 352 353 /* 354 * Fence debugfs 355 */ 356 #if defined(CONFIG_DEBUG_FS) 357 static int radeon_debugfs_fence_info(struct seq_file *m, void *data) 358 { 359 struct drm_info_node *node = (struct drm_info_node *)m->private; 360 struct drm_device *dev = node->minor->dev; 361 struct radeon_device *rdev = dev->dev_private; 362 struct radeon_fence *fence; 363 364 seq_printf(m, "Last signaled fence 0x%08X\n", 365 RREG32(rdev->fence_drv.scratch_reg)); 366 if (!list_empty(&rdev->fence_drv.emited)) { 367 fence = list_entry(rdev->fence_drv.emited.prev, 368 struct radeon_fence, list); 369 seq_printf(m, "Last emited fence %p with 0x%08X\n", 370 fence, fence->seq); 371 } 372 return 0; 373 } 374 375 static struct drm_info_list radeon_debugfs_fence_list[] = { 376 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, 377 }; 378 #endif 379 380 int radeon_debugfs_fence_init(struct radeon_device *rdev) 381 { 382 #if defined(CONFIG_DEBUG_FS) 383 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1); 384 #else 385 return 0; 386 #endif 387 } 388