1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/seq_file.h>
29 #include <linux/slab.h>
30 #include "drmP.h"
31 #include "radeon_drm.h"
32 #include "radeon_reg.h"
33 #include "radeon.h"
34 #include "atom.h"
35 
36 int radeon_debugfs_ib_init(struct radeon_device *rdev);
37 int radeon_debugfs_ring_init(struct radeon_device *rdev);
38 
39 u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
40 {
41 	struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
42 	u32 pg_idx, pg_offset;
43 	u32 idx_value = 0;
44 	int new_page;
45 
46 	pg_idx = (idx * 4) / PAGE_SIZE;
47 	pg_offset = (idx * 4) % PAGE_SIZE;
48 
49 	if (ibc->kpage_idx[0] == pg_idx)
50 		return ibc->kpage[0][pg_offset/4];
51 	if (ibc->kpage_idx[1] == pg_idx)
52 		return ibc->kpage[1][pg_offset/4];
53 
54 	new_page = radeon_cs_update_pages(p, pg_idx);
55 	if (new_page < 0) {
56 		p->parser_error = new_page;
57 		return 0;
58 	}
59 
60 	idx_value = ibc->kpage[new_page][pg_offset/4];
61 	return idx_value;
62 }
63 
64 void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
65 {
66 #if DRM_DEBUG_CODE
67 	if (ring->count_dw <= 0) {
68 		DRM_ERROR("radeon: writting more dword to ring than expected !\n");
69 	}
70 #endif
71 	ring->ring[ring->wptr++] = v;
72 	ring->wptr &= ring->ptr_mask;
73 	ring->count_dw--;
74 	ring->ring_free_dw--;
75 }
76 
77 /*
78  * IB.
79  */
80 bool radeon_ib_try_free(struct radeon_device *rdev, struct radeon_ib *ib)
81 {
82 	bool done = false;
83 
84 	/* only free ib which have been emited */
85 	if (ib->fence && ib->fence->emitted) {
86 		if (radeon_fence_signaled(ib->fence)) {
87 			radeon_fence_unref(&ib->fence);
88 			radeon_sa_bo_free(rdev, &ib->sa_bo);
89 			done = true;
90 		}
91 	}
92 	return done;
93 }
94 
95 int radeon_ib_get(struct radeon_device *rdev, int ring,
96 		  struct radeon_ib **ib, unsigned size)
97 {
98 	struct radeon_fence *fence;
99 	unsigned cretry = 0;
100 	int r = 0, i, idx;
101 
102 	*ib = NULL;
103 	/* align size on 256 bytes */
104 	size = ALIGN(size, 256);
105 
106 	r = radeon_fence_create(rdev, &fence, ring);
107 	if (r) {
108 		dev_err(rdev->dev, "failed to create fence for new IB\n");
109 		return r;
110 	}
111 
112 	radeon_mutex_lock(&rdev->ib_pool.mutex);
113 	idx = rdev->ib_pool.head_id;
114 retry:
115 	if (cretry > 5) {
116 		dev_err(rdev->dev, "failed to get an ib after 5 retry\n");
117 		radeon_mutex_unlock(&rdev->ib_pool.mutex);
118 		radeon_fence_unref(&fence);
119 		return -ENOMEM;
120 	}
121 	cretry++;
122 	for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
123 		radeon_ib_try_free(rdev, &rdev->ib_pool.ibs[idx]);
124 		if (rdev->ib_pool.ibs[idx].fence == NULL) {
125 			r = radeon_sa_bo_new(rdev, &rdev->ib_pool.sa_manager,
126 					     &rdev->ib_pool.ibs[idx].sa_bo,
127 					     size, 256);
128 			if (!r) {
129 				*ib = &rdev->ib_pool.ibs[idx];
130 				(*ib)->ptr = rdev->ib_pool.sa_manager.cpu_ptr;
131 				(*ib)->ptr += ((*ib)->sa_bo.offset >> 2);
132 				(*ib)->gpu_addr = rdev->ib_pool.sa_manager.gpu_addr;
133 				(*ib)->gpu_addr += (*ib)->sa_bo.offset;
134 				(*ib)->fence = fence;
135 				(*ib)->vm_id = 0;
136 				/* ib are most likely to be allocated in a ring fashion
137 				 * thus rdev->ib_pool.head_id should be the id of the
138 				 * oldest ib
139 				 */
140 				rdev->ib_pool.head_id = (1 + idx);
141 				rdev->ib_pool.head_id &= (RADEON_IB_POOL_SIZE - 1);
142 				radeon_mutex_unlock(&rdev->ib_pool.mutex);
143 				return 0;
144 			}
145 		}
146 		idx = (idx + 1) & (RADEON_IB_POOL_SIZE - 1);
147 	}
148 	/* this should be rare event, ie all ib scheduled none signaled yet.
149 	 */
150 	for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
151 		if (rdev->ib_pool.ibs[idx].fence && rdev->ib_pool.ibs[idx].fence->emitted) {
152 			r = radeon_fence_wait(rdev->ib_pool.ibs[idx].fence, false);
153 			if (!r) {
154 				goto retry;
155 			}
156 			/* an error happened */
157 			break;
158 		}
159 		idx = (idx + 1) & (RADEON_IB_POOL_SIZE - 1);
160 	}
161 	radeon_mutex_unlock(&rdev->ib_pool.mutex);
162 	radeon_fence_unref(&fence);
163 	return r;
164 }
165 
166 void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib)
167 {
168 	struct radeon_ib *tmp = *ib;
169 
170 	*ib = NULL;
171 	if (tmp == NULL) {
172 		return;
173 	}
174 	radeon_mutex_lock(&rdev->ib_pool.mutex);
175 	if (tmp->fence && !tmp->fence->emitted) {
176 		radeon_sa_bo_free(rdev, &tmp->sa_bo);
177 		radeon_fence_unref(&tmp->fence);
178 	}
179 	radeon_mutex_unlock(&rdev->ib_pool.mutex);
180 }
181 
182 int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)
183 {
184 	struct radeon_ring *ring = &rdev->ring[ib->fence->ring];
185 	int r = 0;
186 
187 	if (!ib->length_dw || !ring->ready) {
188 		/* TODO: Nothings in the ib we should report. */
189 		DRM_ERROR("radeon: couldn't schedule IB(%u).\n", ib->idx);
190 		return -EINVAL;
191 	}
192 
193 	/* 64 dwords should be enough for fence too */
194 	r = radeon_ring_lock(rdev, ring, 64);
195 	if (r) {
196 		DRM_ERROR("radeon: scheduling IB failed (%d).\n", r);
197 		return r;
198 	}
199 	radeon_ring_ib_execute(rdev, ib->fence->ring, ib);
200 	radeon_fence_emit(rdev, ib->fence);
201 	radeon_ring_unlock_commit(rdev, ring);
202 	return 0;
203 }
204 
205 int radeon_ib_pool_init(struct radeon_device *rdev)
206 {
207 	struct radeon_sa_manager tmp;
208 	int i, r;
209 
210 	r = radeon_sa_bo_manager_init(rdev, &tmp,
211 				      RADEON_IB_POOL_SIZE*64*1024,
212 				      RADEON_GEM_DOMAIN_GTT);
213 	if (r) {
214 		return r;
215 	}
216 
217 	radeon_mutex_lock(&rdev->ib_pool.mutex);
218 	if (rdev->ib_pool.ready) {
219 		radeon_mutex_unlock(&rdev->ib_pool.mutex);
220 		radeon_sa_bo_manager_fini(rdev, &tmp);
221 		return 0;
222 	}
223 
224 	rdev->ib_pool.sa_manager = tmp;
225 	INIT_LIST_HEAD(&rdev->ib_pool.sa_manager.sa_bo);
226 	for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
227 		rdev->ib_pool.ibs[i].fence = NULL;
228 		rdev->ib_pool.ibs[i].idx = i;
229 		rdev->ib_pool.ibs[i].length_dw = 0;
230 		INIT_LIST_HEAD(&rdev->ib_pool.ibs[i].sa_bo.list);
231 	}
232 	rdev->ib_pool.head_id = 0;
233 	rdev->ib_pool.ready = true;
234 	DRM_INFO("radeon: ib pool ready.\n");
235 
236 	if (radeon_debugfs_ib_init(rdev)) {
237 		DRM_ERROR("Failed to register debugfs file for IB !\n");
238 	}
239 	if (radeon_debugfs_ring_init(rdev)) {
240 		DRM_ERROR("Failed to register debugfs file for rings !\n");
241 	}
242 	radeon_mutex_unlock(&rdev->ib_pool.mutex);
243 	return 0;
244 }
245 
246 void radeon_ib_pool_fini(struct radeon_device *rdev)
247 {
248 	unsigned i;
249 
250 	radeon_mutex_lock(&rdev->ib_pool.mutex);
251 	if (rdev->ib_pool.ready) {
252 		for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
253 			radeon_sa_bo_free(rdev, &rdev->ib_pool.ibs[i].sa_bo);
254 			radeon_fence_unref(&rdev->ib_pool.ibs[i].fence);
255 		}
256 		radeon_sa_bo_manager_fini(rdev, &rdev->ib_pool.sa_manager);
257 		rdev->ib_pool.ready = false;
258 	}
259 	radeon_mutex_unlock(&rdev->ib_pool.mutex);
260 }
261 
262 int radeon_ib_pool_start(struct radeon_device *rdev)
263 {
264 	return radeon_sa_bo_manager_start(rdev, &rdev->ib_pool.sa_manager);
265 }
266 
267 int radeon_ib_pool_suspend(struct radeon_device *rdev)
268 {
269 	return radeon_sa_bo_manager_suspend(rdev, &rdev->ib_pool.sa_manager);
270 }
271 
272 /*
273  * Ring.
274  */
275 int radeon_ring_index(struct radeon_device *rdev, struct radeon_ring *ring)
276 {
277 	/* r1xx-r5xx only has CP ring */
278 	if (rdev->family < CHIP_R600)
279 		return RADEON_RING_TYPE_GFX_INDEX;
280 
281 	if (rdev->family >= CHIP_CAYMAN) {
282 		if (ring == &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX])
283 			return CAYMAN_RING_TYPE_CP1_INDEX;
284 		else if (ring == &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX])
285 			return CAYMAN_RING_TYPE_CP2_INDEX;
286 	}
287 	return RADEON_RING_TYPE_GFX_INDEX;
288 }
289 
290 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
291 {
292 	u32 rptr;
293 
294 	if (rdev->wb.enabled)
295 		rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
296 	else
297 		rptr = RREG32(ring->rptr_reg);
298 	ring->rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
299 	/* This works because ring_size is a power of 2 */
300 	ring->ring_free_dw = (ring->rptr + (ring->ring_size / 4));
301 	ring->ring_free_dw -= ring->wptr;
302 	ring->ring_free_dw &= ring->ptr_mask;
303 	if (!ring->ring_free_dw) {
304 		ring->ring_free_dw = ring->ring_size / 4;
305 	}
306 }
307 
308 
309 int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
310 {
311 	int r;
312 
313 	/* Align requested size with padding so unlock_commit can
314 	 * pad safely */
315 	ndw = (ndw + ring->align_mask) & ~ring->align_mask;
316 	while (ndw > (ring->ring_free_dw - 1)) {
317 		radeon_ring_free_size(rdev, ring);
318 		if (ndw < ring->ring_free_dw) {
319 			break;
320 		}
321 		r = radeon_fence_wait_next(rdev, radeon_ring_index(rdev, ring));
322 		if (r)
323 			return r;
324 	}
325 	ring->count_dw = ndw;
326 	ring->wptr_old = ring->wptr;
327 	return 0;
328 }
329 
330 int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
331 {
332 	int r;
333 
334 	mutex_lock(&ring->mutex);
335 	r = radeon_ring_alloc(rdev, ring, ndw);
336 	if (r) {
337 		mutex_unlock(&ring->mutex);
338 		return r;
339 	}
340 	return 0;
341 }
342 
343 void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring)
344 {
345 	unsigned count_dw_pad;
346 	unsigned i;
347 
348 	/* We pad to match fetch size */
349 	count_dw_pad = (ring->align_mask + 1) -
350 		       (ring->wptr & ring->align_mask);
351 	for (i = 0; i < count_dw_pad; i++) {
352 		radeon_ring_write(ring, ring->nop);
353 	}
354 	DRM_MEMORYBARRIER();
355 	WREG32(ring->wptr_reg, (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask);
356 	(void)RREG32(ring->wptr_reg);
357 }
358 
359 void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring)
360 {
361 	radeon_ring_commit(rdev, ring);
362 	mutex_unlock(&ring->mutex);
363 }
364 
365 void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)
366 {
367 	ring->wptr = ring->wptr_old;
368 	mutex_unlock(&ring->mutex);
369 }
370 
371 int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,
372 		     unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg,
373 		     u32 ptr_reg_shift, u32 ptr_reg_mask, u32 nop)
374 {
375 	int r;
376 
377 	ring->ring_size = ring_size;
378 	ring->rptr_offs = rptr_offs;
379 	ring->rptr_reg = rptr_reg;
380 	ring->wptr_reg = wptr_reg;
381 	ring->ptr_reg_shift = ptr_reg_shift;
382 	ring->ptr_reg_mask = ptr_reg_mask;
383 	ring->nop = nop;
384 	/* Allocate ring buffer */
385 	if (ring->ring_obj == NULL) {
386 		r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,
387 					RADEON_GEM_DOMAIN_GTT,
388 					&ring->ring_obj);
389 		if (r) {
390 			dev_err(rdev->dev, "(%d) ring create failed\n", r);
391 			return r;
392 		}
393 		r = radeon_bo_reserve(ring->ring_obj, false);
394 		if (unlikely(r != 0))
395 			return r;
396 		r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,
397 					&ring->gpu_addr);
398 		if (r) {
399 			radeon_bo_unreserve(ring->ring_obj);
400 			dev_err(rdev->dev, "(%d) ring pin failed\n", r);
401 			return r;
402 		}
403 		r = radeon_bo_kmap(ring->ring_obj,
404 				       (void **)&ring->ring);
405 		radeon_bo_unreserve(ring->ring_obj);
406 		if (r) {
407 			dev_err(rdev->dev, "(%d) ring map failed\n", r);
408 			return r;
409 		}
410 	}
411 	ring->ptr_mask = (ring->ring_size / 4) - 1;
412 	ring->ring_free_dw = ring->ring_size / 4;
413 	return 0;
414 }
415 
416 void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
417 {
418 	int r;
419 	struct radeon_bo *ring_obj;
420 
421 	mutex_lock(&ring->mutex);
422 	ring_obj = ring->ring_obj;
423 	ring->ring = NULL;
424 	ring->ring_obj = NULL;
425 	mutex_unlock(&ring->mutex);
426 
427 	if (ring_obj) {
428 		r = radeon_bo_reserve(ring_obj, false);
429 		if (likely(r == 0)) {
430 			radeon_bo_kunmap(ring_obj);
431 			radeon_bo_unpin(ring_obj);
432 			radeon_bo_unreserve(ring_obj);
433 		}
434 		radeon_bo_unref(&ring_obj);
435 	}
436 }
437 
438 /*
439  * Debugfs info
440  */
441 #if defined(CONFIG_DEBUG_FS)
442 
443 static int radeon_debugfs_ring_info(struct seq_file *m, void *data)
444 {
445 	struct drm_info_node *node = (struct drm_info_node *) m->private;
446 	struct drm_device *dev = node->minor->dev;
447 	struct radeon_device *rdev = dev->dev_private;
448 	int ridx = *(int*)node->info_ent->data;
449 	struct radeon_ring *ring = &rdev->ring[ridx];
450 	unsigned count, i, j;
451 
452 	radeon_ring_free_size(rdev, ring);
453 	count = (ring->ring_size / 4) - ring->ring_free_dw;
454 	seq_printf(m, "wptr(0x%04x): 0x%08x\n", ring->wptr_reg, RREG32(ring->wptr_reg));
455 	seq_printf(m, "rptr(0x%04x): 0x%08x\n", ring->rptr_reg, RREG32(ring->rptr_reg));
456 	seq_printf(m, "driver's copy of the wptr: 0x%08x\n", ring->wptr);
457 	seq_printf(m, "driver's copy of the rptr: 0x%08x\n", ring->rptr);
458 	seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
459 	seq_printf(m, "%u dwords in ring\n", count);
460 	i = ring->rptr;
461 	for (j = 0; j <= count; j++) {
462 		seq_printf(m, "r[%04d]=0x%08x\n", i, ring->ring[i]);
463 		i = (i + 1) & ring->ptr_mask;
464 	}
465 	return 0;
466 }
467 
468 static int radeon_ring_type_gfx_index = RADEON_RING_TYPE_GFX_INDEX;
469 static int cayman_ring_type_cp1_index = CAYMAN_RING_TYPE_CP1_INDEX;
470 static int cayman_ring_type_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX;
471 
472 static struct drm_info_list radeon_debugfs_ring_info_list[] = {
473 	{"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_ring_type_gfx_index},
474 	{"radeon_ring_cp1", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp1_index},
475 	{"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp2_index},
476 };
477 
478 static int radeon_debugfs_ib_info(struct seq_file *m, void *data)
479 {
480 	struct drm_info_node *node = (struct drm_info_node *) m->private;
481 	struct radeon_ib *ib = node->info_ent->data;
482 	unsigned i;
483 
484 	if (ib == NULL) {
485 		return 0;
486 	}
487 	seq_printf(m, "IB %04u\n", ib->idx);
488 	seq_printf(m, "IB fence %p\n", ib->fence);
489 	seq_printf(m, "IB size %05u dwords\n", ib->length_dw);
490 	for (i = 0; i < ib->length_dw; i++) {
491 		seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]);
492 	}
493 	return 0;
494 }
495 
496 static struct drm_info_list radeon_debugfs_ib_list[RADEON_IB_POOL_SIZE];
497 static char radeon_debugfs_ib_names[RADEON_IB_POOL_SIZE][32];
498 #endif
499 
500 int radeon_debugfs_ring_init(struct radeon_device *rdev)
501 {
502 #if defined(CONFIG_DEBUG_FS)
503 	if (rdev->family >= CHIP_CAYMAN)
504 		return radeon_debugfs_add_files(rdev, radeon_debugfs_ring_info_list,
505 						ARRAY_SIZE(radeon_debugfs_ring_info_list));
506 	else
507 		return radeon_debugfs_add_files(rdev, radeon_debugfs_ring_info_list, 1);
508 #else
509 	return 0;
510 #endif
511 }
512 
513 int radeon_debugfs_ib_init(struct radeon_device *rdev)
514 {
515 #if defined(CONFIG_DEBUG_FS)
516 	unsigned i;
517 
518 	for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
519 		sprintf(radeon_debugfs_ib_names[i], "radeon_ib_%04u", i);
520 		radeon_debugfs_ib_list[i].name = radeon_debugfs_ib_names[i];
521 		radeon_debugfs_ib_list[i].show = &radeon_debugfs_ib_info;
522 		radeon_debugfs_ib_list[i].driver_features = 0;
523 		radeon_debugfs_ib_list[i].data = &rdev->ib_pool.ibs[i];
524 	}
525 	return radeon_debugfs_add_files(rdev, radeon_debugfs_ib_list,
526 					RADEON_IB_POOL_SIZE);
527 #else
528 	return 0;
529 #endif
530 }
531