1 /*
2  * Copyright 2013 Advanced Micro Devices, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  * Authors: Christian König <christian.koenig@amd.com>
26  */
27 
28 #include <linux/firmware.h>
29 #include <linux/module.h>
30 
31 #include <drm/drm.h>
32 
33 #include "radeon.h"
34 #include "radeon_asic.h"
35 #include "sid.h"
36 
37 /* 1 second timeout */
38 #define VCE_IDLE_TIMEOUT_MS	1000
39 
40 /* Firmware Names */
41 #define FIRMWARE_TAHITI	"radeon/TAHITI_vce.bin"
42 #define FIRMWARE_BONAIRE	"radeon/BONAIRE_vce.bin"
43 
44 MODULE_FIRMWARE(FIRMWARE_TAHITI);
45 MODULE_FIRMWARE(FIRMWARE_BONAIRE);
46 
47 static void radeon_vce_idle_work_handler(struct work_struct *work);
48 
49 /**
50  * radeon_vce_init - allocate memory, load vce firmware
51  *
52  * @rdev: radeon_device pointer
53  *
54  * First step to get VCE online, allocate memory and load the firmware
55  */
56 int radeon_vce_init(struct radeon_device *rdev)
57 {
58 	static const char *fw_version = "[ATI LIB=VCEFW,";
59 	static const char *fb_version = "[ATI LIB=VCEFWSTATS,";
60 	unsigned long size;
61 	const char *fw_name, *c;
62 	uint8_t start, mid, end;
63 	int i, r;
64 
65 	INIT_DELAYED_WORK(&rdev->vce.idle_work, radeon_vce_idle_work_handler);
66 
67 	switch (rdev->family) {
68 	case CHIP_TAHITI:
69 	case CHIP_PITCAIRN:
70 	case CHIP_VERDE:
71 	case CHIP_ARUBA:
72 		fw_name = FIRMWARE_TAHITI;
73 		break;
74 
75 	case CHIP_BONAIRE:
76 	case CHIP_KAVERI:
77 	case CHIP_KABINI:
78 	case CHIP_HAWAII:
79 	case CHIP_MULLINS:
80 		fw_name = FIRMWARE_BONAIRE;
81 		break;
82 
83 	default:
84 		return -EINVAL;
85 	}
86 
87 	r = request_firmware(&rdev->vce_fw, fw_name, rdev->dev);
88 	if (r) {
89 		dev_err(rdev->dev, "radeon_vce: Can't load firmware \"%s\"\n",
90 			fw_name);
91 		return r;
92 	}
93 
94 	/* search for firmware version */
95 
96 	size = rdev->vce_fw->size - strlen(fw_version) - 9;
97 	c = rdev->vce_fw->data;
98 	for (; size > 0; --size, ++c)
99 		if (strncmp(c, fw_version, strlen(fw_version)) == 0)
100 			break;
101 
102 	if (size == 0)
103 		return -EINVAL;
104 
105 	c += strlen(fw_version);
106 	if (sscanf(c, "%2hhd.%2hhd.%2hhd]", &start, &mid, &end) != 3)
107 		return -EINVAL;
108 
109 	/* search for feedback version */
110 
111 	size = rdev->vce_fw->size - strlen(fb_version) - 3;
112 	c = rdev->vce_fw->data;
113 	for (; size > 0; --size, ++c)
114 		if (strncmp(c, fb_version, strlen(fb_version)) == 0)
115 			break;
116 
117 	if (size == 0)
118 		return -EINVAL;
119 
120 	c += strlen(fb_version);
121 	if (sscanf(c, "%2u]", &rdev->vce.fb_version) != 1)
122 		return -EINVAL;
123 
124 	DRM_INFO("Found VCE firmware/feedback version %d.%d.%d / %d!\n",
125 		 start, mid, end, rdev->vce.fb_version);
126 
127 	rdev->vce.fw_version = (start << 24) | (mid << 16) | (end << 8);
128 
129 	/* we can only work with this fw version for now */
130 	if ((rdev->vce.fw_version != ((40 << 24) | (2 << 16) | (2 << 8))) &&
131 	    (rdev->vce.fw_version != ((50 << 24) | (0 << 16) | (1 << 8))) &&
132 	    (rdev->vce.fw_version != ((50 << 24) | (1 << 16) | (2 << 8))))
133 		return -EINVAL;
134 
135 	/* allocate firmware, stack and heap BO */
136 
137 	if (rdev->family < CHIP_BONAIRE)
138 		size = vce_v1_0_bo_size(rdev);
139 	else
140 		size = vce_v2_0_bo_size(rdev);
141 	r = radeon_bo_create(rdev, size, PAGE_SIZE, true,
142 			     RADEON_GEM_DOMAIN_VRAM, 0, NULL, NULL,
143 			     &rdev->vce.vcpu_bo);
144 	if (r) {
145 		dev_err(rdev->dev, "(%d) failed to allocate VCE bo\n", r);
146 		return r;
147 	}
148 
149 	r = radeon_bo_reserve(rdev->vce.vcpu_bo, false);
150 	if (r) {
151 		radeon_bo_unref(&rdev->vce.vcpu_bo);
152 		dev_err(rdev->dev, "(%d) failed to reserve VCE bo\n", r);
153 		return r;
154 	}
155 
156 	r = radeon_bo_pin(rdev->vce.vcpu_bo, RADEON_GEM_DOMAIN_VRAM,
157 			  &rdev->vce.gpu_addr);
158 	radeon_bo_unreserve(rdev->vce.vcpu_bo);
159 	if (r) {
160 		radeon_bo_unref(&rdev->vce.vcpu_bo);
161 		dev_err(rdev->dev, "(%d) VCE bo pin failed\n", r);
162 		return r;
163 	}
164 
165 	for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
166 		atomic_set(&rdev->vce.handles[i], 0);
167 		rdev->vce.filp[i] = NULL;
168 	}
169 
170 	return 0;
171 }
172 
173 /**
174  * radeon_vce_fini - free memory
175  *
176  * @rdev: radeon_device pointer
177  *
178  * Last step on VCE teardown, free firmware memory
179  */
180 void radeon_vce_fini(struct radeon_device *rdev)
181 {
182 	if (rdev->vce.vcpu_bo == NULL)
183 		return;
184 
185 	radeon_bo_unref(&rdev->vce.vcpu_bo);
186 
187 	release_firmware(rdev->vce_fw);
188 }
189 
190 /**
191  * radeon_vce_suspend - unpin VCE fw memory
192  *
193  * @rdev: radeon_device pointer
194  *
195  */
196 int radeon_vce_suspend(struct radeon_device *rdev)
197 {
198 	int i;
199 
200 	if (rdev->vce.vcpu_bo == NULL)
201 		return 0;
202 
203 	for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i)
204 		if (atomic_read(&rdev->vce.handles[i]))
205 			break;
206 
207 	if (i == RADEON_MAX_VCE_HANDLES)
208 		return 0;
209 
210 	/* TODO: suspending running encoding sessions isn't supported */
211 	return -EINVAL;
212 }
213 
214 /**
215  * radeon_vce_resume - pin VCE fw memory
216  *
217  * @rdev: radeon_device pointer
218  *
219  */
220 int radeon_vce_resume(struct radeon_device *rdev)
221 {
222 	void *cpu_addr;
223 	int r;
224 
225 	if (rdev->vce.vcpu_bo == NULL)
226 		return -EINVAL;
227 
228 	r = radeon_bo_reserve(rdev->vce.vcpu_bo, false);
229 	if (r) {
230 		dev_err(rdev->dev, "(%d) failed to reserve VCE bo\n", r);
231 		return r;
232 	}
233 
234 	r = radeon_bo_kmap(rdev->vce.vcpu_bo, &cpu_addr);
235 	if (r) {
236 		radeon_bo_unreserve(rdev->vce.vcpu_bo);
237 		dev_err(rdev->dev, "(%d) VCE map failed\n", r);
238 		return r;
239 	}
240 
241 	memset(cpu_addr, 0, radeon_bo_size(rdev->vce.vcpu_bo));
242 	if (rdev->family < CHIP_BONAIRE)
243 		r = vce_v1_0_load_fw(rdev, cpu_addr);
244 	else
245 		memcpy(cpu_addr, rdev->vce_fw->data, rdev->vce_fw->size);
246 
247 	radeon_bo_kunmap(rdev->vce.vcpu_bo);
248 
249 	radeon_bo_unreserve(rdev->vce.vcpu_bo);
250 
251 	return r;
252 }
253 
254 /**
255  * radeon_vce_idle_work_handler - power off VCE
256  *
257  * @work: pointer to work structure
258  *
259  * power of VCE when it's not used any more
260  */
261 static void radeon_vce_idle_work_handler(struct work_struct *work)
262 {
263 	struct radeon_device *rdev =
264 		container_of(work, struct radeon_device, vce.idle_work.work);
265 
266 	if ((radeon_fence_count_emitted(rdev, TN_RING_TYPE_VCE1_INDEX) == 0) &&
267 	    (radeon_fence_count_emitted(rdev, TN_RING_TYPE_VCE2_INDEX) == 0)) {
268 		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
269 			radeon_dpm_enable_vce(rdev, false);
270 		} else {
271 			radeon_set_vce_clocks(rdev, 0, 0);
272 		}
273 	} else {
274 		schedule_delayed_work(&rdev->vce.idle_work,
275 				      msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS));
276 	}
277 }
278 
279 /**
280  * radeon_vce_note_usage - power up VCE
281  *
282  * @rdev: radeon_device pointer
283  *
284  * Make sure VCE is powerd up when we want to use it
285  */
286 void radeon_vce_note_usage(struct radeon_device *rdev)
287 {
288 	bool streams_changed = false;
289 	bool set_clocks = !cancel_delayed_work_sync(&rdev->vce.idle_work);
290 	set_clocks &= schedule_delayed_work(&rdev->vce.idle_work,
291 					    msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS));
292 
293 	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
294 		/* XXX figure out if the streams changed */
295 		streams_changed = false;
296 	}
297 
298 	if (set_clocks || streams_changed) {
299 		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
300 			radeon_dpm_enable_vce(rdev, true);
301 		} else {
302 			radeon_set_vce_clocks(rdev, 53300, 40000);
303 		}
304 	}
305 }
306 
307 /**
308  * radeon_vce_free_handles - free still open VCE handles
309  *
310  * @rdev: radeon_device pointer
311  * @filp: drm file pointer
312  *
313  * Close all VCE handles still open by this file pointer
314  */
315 void radeon_vce_free_handles(struct radeon_device *rdev, struct drm_file *filp)
316 {
317 	int i, r;
318 	for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
319 		uint32_t handle = atomic_read(&rdev->vce.handles[i]);
320 		if (!handle || rdev->vce.filp[i] != filp)
321 			continue;
322 
323 		radeon_vce_note_usage(rdev);
324 
325 		r = radeon_vce_get_destroy_msg(rdev, TN_RING_TYPE_VCE1_INDEX,
326 					       handle, NULL);
327 		if (r)
328 			DRM_ERROR("Error destroying VCE handle (%d)!\n", r);
329 
330 		rdev->vce.filp[i] = NULL;
331 		atomic_set(&rdev->vce.handles[i], 0);
332 	}
333 }
334 
335 /**
336  * radeon_vce_get_create_msg - generate a VCE create msg
337  *
338  * @rdev: radeon_device pointer
339  * @ring: ring we should submit the msg to
340  * @handle: VCE session handle to use
341  * @fence: optional fence to return
342  *
343  * Open up a stream for HW test
344  */
345 int radeon_vce_get_create_msg(struct radeon_device *rdev, int ring,
346 			      uint32_t handle, struct radeon_fence **fence)
347 {
348 	const unsigned ib_size_dw = 1024;
349 	struct radeon_ib ib;
350 	uint64_t dummy;
351 	int i, r;
352 
353 	r = radeon_ib_get(rdev, ring, &ib, NULL, ib_size_dw * 4);
354 	if (r) {
355 		DRM_ERROR("radeon: failed to get ib (%d).\n", r);
356 		return r;
357 	}
358 
359 	dummy = ib.gpu_addr + 1024;
360 
361 	/* stitch together an VCE create msg */
362 	ib.length_dw = 0;
363 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x0000000c); /* len */
364 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001); /* session cmd */
365 	ib.ptr[ib.length_dw++] = cpu_to_le32(handle);
366 
367 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000030); /* len */
368 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x01000001); /* create cmd */
369 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000000);
370 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000042);
371 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x0000000a);
372 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001);
373 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000080);
374 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000060);
375 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000100);
376 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000100);
377 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x0000000c);
378 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000000);
379 
380 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000014); /* len */
381 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x05000005); /* feedback buffer */
382 	ib.ptr[ib.length_dw++] = cpu_to_le32(upper_32_bits(dummy));
383 	ib.ptr[ib.length_dw++] = cpu_to_le32(dummy);
384 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001);
385 
386 	for (i = ib.length_dw; i < ib_size_dw; ++i)
387 		ib.ptr[i] = cpu_to_le32(0x0);
388 
389 	r = radeon_ib_schedule(rdev, &ib, NULL, false);
390 	if (r)
391 		DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
392 
393 
394 	if (fence)
395 		*fence = radeon_fence_ref(ib.fence);
396 
397 	radeon_ib_free(rdev, &ib);
398 
399 	return r;
400 }
401 
402 /**
403  * radeon_vce_get_destroy_msg - generate a VCE destroy msg
404  *
405  * @rdev: radeon_device pointer
406  * @ring: ring we should submit the msg to
407  * @handle: VCE session handle to use
408  * @fence: optional fence to return
409  *
410  * Close up a stream for HW test or if userspace failed to do so
411  */
412 int radeon_vce_get_destroy_msg(struct radeon_device *rdev, int ring,
413 			       uint32_t handle, struct radeon_fence **fence)
414 {
415 	const unsigned ib_size_dw = 1024;
416 	struct radeon_ib ib;
417 	uint64_t dummy;
418 	int i, r;
419 
420 	r = radeon_ib_get(rdev, ring, &ib, NULL, ib_size_dw * 4);
421 	if (r) {
422 		DRM_ERROR("radeon: failed to get ib (%d).\n", r);
423 		return r;
424 	}
425 
426 	dummy = ib.gpu_addr + 1024;
427 
428 	/* stitch together an VCE destroy msg */
429 	ib.length_dw = 0;
430 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x0000000c); /* len */
431 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001); /* session cmd */
432 	ib.ptr[ib.length_dw++] = cpu_to_le32(handle);
433 
434 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000014); /* len */
435 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x05000005); /* feedback buffer */
436 	ib.ptr[ib.length_dw++] = cpu_to_le32(upper_32_bits(dummy));
437 	ib.ptr[ib.length_dw++] = cpu_to_le32(dummy);
438 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000001);
439 
440 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x00000008); /* len */
441 	ib.ptr[ib.length_dw++] = cpu_to_le32(0x02000001); /* destroy cmd */
442 
443 	for (i = ib.length_dw; i < ib_size_dw; ++i)
444 		ib.ptr[i] = cpu_to_le32(0x0);
445 
446 	r = radeon_ib_schedule(rdev, &ib, NULL, false);
447 	if (r) {
448 		DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
449 	}
450 
451 	if (fence)
452 		*fence = radeon_fence_ref(ib.fence);
453 
454 	radeon_ib_free(rdev, &ib);
455 
456 	return r;
457 }
458 
459 /**
460  * radeon_vce_cs_reloc - command submission relocation
461  *
462  * @p: parser context
463  * @lo: address of lower dword
464  * @hi: address of higher dword
465  * @size: size of checker for relocation buffer
466  *
467  * Patch relocation inside command stream with real buffer address
468  */
469 int radeon_vce_cs_reloc(struct radeon_cs_parser *p, int lo, int hi,
470 			unsigned size)
471 {
472 	struct radeon_cs_chunk *relocs_chunk;
473 	struct radeon_bo_list *reloc;
474 	uint64_t start, end, offset;
475 	unsigned idx;
476 
477 	relocs_chunk = p->chunk_relocs;
478 	offset = radeon_get_ib_value(p, lo);
479 	idx = radeon_get_ib_value(p, hi);
480 
481 	if (idx >= relocs_chunk->length_dw) {
482 		DRM_ERROR("Relocs at %d after relocations chunk end %d !\n",
483 			  idx, relocs_chunk->length_dw);
484 		return -EINVAL;
485 	}
486 
487 	reloc = &p->relocs[(idx / 4)];
488 	start = reloc->gpu_offset;
489 	end = start + radeon_bo_size(reloc->robj);
490 	start += offset;
491 
492 	p->ib.ptr[lo] = start & 0xFFFFFFFF;
493 	p->ib.ptr[hi] = start >> 32;
494 
495 	if (end <= start) {
496 		DRM_ERROR("invalid reloc offset %llX!\n", offset);
497 		return -EINVAL;
498 	}
499 	if ((end - start) < size) {
500 		DRM_ERROR("buffer to small (%d / %d)!\n",
501 			(unsigned)(end - start), size);
502 		return -EINVAL;
503 	}
504 
505 	return 0;
506 }
507 
508 /**
509  * radeon_vce_validate_handle - validate stream handle
510  *
511  * @p: parser context
512  * @handle: handle to validate
513  * @allocated: allocated a new handle?
514  *
515  * Validates the handle and return the found session index or -EINVAL
516  * we don't have another free session index.
517  */
518 static int radeon_vce_validate_handle(struct radeon_cs_parser *p,
519 				      uint32_t handle, bool *allocated)
520 {
521 	unsigned i;
522 
523 	*allocated = false;
524 
525 	/* validate the handle */
526 	for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
527 		if (atomic_read(&p->rdev->vce.handles[i]) == handle) {
528 			if (p->rdev->vce.filp[i] != p->filp) {
529 				DRM_ERROR("VCE handle collision detected!\n");
530 				return -EINVAL;
531 			}
532 			return i;
533 		}
534 	}
535 
536 	/* handle not found try to alloc a new one */
537 	for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
538 		if (!atomic_cmpxchg(&p->rdev->vce.handles[i], 0, handle)) {
539 			p->rdev->vce.filp[i] = p->filp;
540 			p->rdev->vce.img_size[i] = 0;
541 			*allocated = true;
542 			return i;
543 		}
544 	}
545 
546 	DRM_ERROR("No more free VCE handles!\n");
547 	return -EINVAL;
548 }
549 
550 /**
551  * radeon_vce_cs_parse - parse and validate the command stream
552  *
553  * @p: parser context
554  *
555  */
556 int radeon_vce_cs_parse(struct radeon_cs_parser *p)
557 {
558 	int session_idx = -1;
559 	bool destroyed = false, created = false, allocated = false;
560 	uint32_t tmp, handle = 0;
561 	uint32_t *size = &tmp;
562 	int i, r = 0;
563 
564 	while (p->idx < p->chunk_ib->length_dw) {
565 		uint32_t len = radeon_get_ib_value(p, p->idx);
566 		uint32_t cmd = radeon_get_ib_value(p, p->idx + 1);
567 
568 		if ((len < 8) || (len & 3)) {
569 			DRM_ERROR("invalid VCE command length (%d)!\n", len);
570 			r = -EINVAL;
571 			goto out;
572 		}
573 
574 		if (destroyed) {
575 			DRM_ERROR("No other command allowed after destroy!\n");
576 			r = -EINVAL;
577 			goto out;
578 		}
579 
580 		switch (cmd) {
581 		case 0x00000001: // session
582 			handle = radeon_get_ib_value(p, p->idx + 2);
583 			session_idx = radeon_vce_validate_handle(p, handle,
584 								 &allocated);
585 			if (session_idx < 0)
586 				return session_idx;
587 			size = &p->rdev->vce.img_size[session_idx];
588 			break;
589 
590 		case 0x00000002: // task info
591 			break;
592 
593 		case 0x01000001: // create
594 			created = true;
595 			if (!allocated) {
596 				DRM_ERROR("Handle already in use!\n");
597 				r = -EINVAL;
598 				goto out;
599 			}
600 
601 			*size = radeon_get_ib_value(p, p->idx + 8) *
602 				radeon_get_ib_value(p, p->idx + 10) *
603 				8 * 3 / 2;
604 			break;
605 
606 		case 0x04000001: // config extension
607 		case 0x04000002: // pic control
608 		case 0x04000005: // rate control
609 		case 0x04000007: // motion estimation
610 		case 0x04000008: // rdo
611 		case 0x04000009: // vui
612 			break;
613 
614 		case 0x03000001: // encode
615 			r = radeon_vce_cs_reloc(p, p->idx + 10, p->idx + 9,
616 						*size);
617 			if (r)
618 				goto out;
619 
620 			r = radeon_vce_cs_reloc(p, p->idx + 12, p->idx + 11,
621 						*size / 3);
622 			if (r)
623 				goto out;
624 			break;
625 
626 		case 0x02000001: // destroy
627 			destroyed = true;
628 			break;
629 
630 		case 0x05000001: // context buffer
631 			r = radeon_vce_cs_reloc(p, p->idx + 3, p->idx + 2,
632 						*size * 2);
633 			if (r)
634 				goto out;
635 			break;
636 
637 		case 0x05000004: // video bitstream buffer
638 			tmp = radeon_get_ib_value(p, p->idx + 4);
639 			r = radeon_vce_cs_reloc(p, p->idx + 3, p->idx + 2,
640 						tmp);
641 			if (r)
642 				goto out;
643 			break;
644 
645 		case 0x05000005: // feedback buffer
646 			r = radeon_vce_cs_reloc(p, p->idx + 3, p->idx + 2,
647 						4096);
648 			if (r)
649 				goto out;
650 			break;
651 
652 		default:
653 			DRM_ERROR("invalid VCE command (0x%x)!\n", cmd);
654 			r = -EINVAL;
655 			goto out;
656 		}
657 
658 		if (session_idx == -1) {
659 			DRM_ERROR("no session command at start of IB\n");
660 			r = -EINVAL;
661 			goto out;
662 		}
663 
664 		p->idx += len / 4;
665 	}
666 
667 	if (allocated && !created) {
668 		DRM_ERROR("New session without create command!\n");
669 		r = -ENOENT;
670 	}
671 
672 out:
673 	if ((!r && destroyed) || (r && allocated)) {
674 		/*
675 		 * IB contains a destroy msg or we have allocated an
676 		 * handle and got an error, anyway free the handle
677 		 */
678 		for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i)
679 			atomic_cmpxchg(&p->rdev->vce.handles[i], handle, 0);
680 	}
681 
682 	return r;
683 }
684 
685 /**
686  * radeon_vce_semaphore_emit - emit a semaphore command
687  *
688  * @rdev: radeon_device pointer
689  * @ring: engine to use
690  * @semaphore: address of semaphore
691  * @emit_wait: true=emit wait, false=emit signal
692  *
693  */
694 bool radeon_vce_semaphore_emit(struct radeon_device *rdev,
695 			       struct radeon_ring *ring,
696 			       struct radeon_semaphore *semaphore,
697 			       bool emit_wait)
698 {
699 	uint64_t addr = semaphore->gpu_addr;
700 
701 	radeon_ring_write(ring, cpu_to_le32(VCE_CMD_SEMAPHORE));
702 	radeon_ring_write(ring, cpu_to_le32((addr >> 3) & 0x000FFFFF));
703 	radeon_ring_write(ring, cpu_to_le32((addr >> 23) & 0x000FFFFF));
704 	radeon_ring_write(ring, cpu_to_le32(0x01003000 | (emit_wait ? 1 : 0)));
705 	if (!emit_wait)
706 		radeon_ring_write(ring, cpu_to_le32(VCE_CMD_END));
707 
708 	return true;
709 }
710 
711 /**
712  * radeon_vce_ib_execute - execute indirect buffer
713  *
714  * @rdev: radeon_device pointer
715  * @ib: the IB to execute
716  *
717  */
718 void radeon_vce_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
719 {
720 	struct radeon_ring *ring = &rdev->ring[ib->ring];
721 	radeon_ring_write(ring, cpu_to_le32(VCE_CMD_IB));
722 	radeon_ring_write(ring, cpu_to_le32(ib->gpu_addr));
723 	radeon_ring_write(ring, cpu_to_le32(upper_32_bits(ib->gpu_addr)));
724 	radeon_ring_write(ring, cpu_to_le32(ib->length_dw));
725 }
726 
727 /**
728  * radeon_vce_fence_emit - add a fence command to the ring
729  *
730  * @rdev: radeon_device pointer
731  * @fence: the fence
732  *
733  */
734 void radeon_vce_fence_emit(struct radeon_device *rdev,
735 			   struct radeon_fence *fence)
736 {
737 	struct radeon_ring *ring = &rdev->ring[fence->ring];
738 	uint64_t addr = rdev->fence_drv[fence->ring].gpu_addr;
739 
740 	radeon_ring_write(ring, cpu_to_le32(VCE_CMD_FENCE));
741 	radeon_ring_write(ring, cpu_to_le32(addr));
742 	radeon_ring_write(ring, cpu_to_le32(upper_32_bits(addr)));
743 	radeon_ring_write(ring, cpu_to_le32(fence->seq));
744 	radeon_ring_write(ring, cpu_to_le32(VCE_CMD_TRAP));
745 	radeon_ring_write(ring, cpu_to_le32(VCE_CMD_END));
746 }
747 
748 /**
749  * radeon_vce_ring_test - test if VCE ring is working
750  *
751  * @rdev: radeon_device pointer
752  * @ring: the engine to test on
753  *
754  */
755 int radeon_vce_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
756 {
757 	uint32_t rptr = vce_v1_0_get_rptr(rdev, ring);
758 	unsigned i;
759 	int r;
760 
761 	r = radeon_ring_lock(rdev, ring, 16);
762 	if (r) {
763 		DRM_ERROR("radeon: vce failed to lock ring %d (%d).\n",
764 			  ring->idx, r);
765 		return r;
766 	}
767 	radeon_ring_write(ring, cpu_to_le32(VCE_CMD_END));
768 	radeon_ring_unlock_commit(rdev, ring, false);
769 
770 	for (i = 0; i < rdev->usec_timeout; i++) {
771 		if (vce_v1_0_get_rptr(rdev, ring) != rptr)
772 			break;
773 		udelay(1);
774 	}
775 
776 	if (i < rdev->usec_timeout) {
777 		DRM_INFO("ring test on %d succeeded in %d usecs\n",
778 			 ring->idx, i);
779 	} else {
780 		DRM_ERROR("radeon: ring %d test failed\n",
781 			 ring->idx);
782 		r = -ETIMEDOUT;
783 	}
784 
785 	return r;
786 }
787 
788 /**
789  * radeon_vce_ib_test - test if VCE IBs are working
790  *
791  * @rdev: radeon_device pointer
792  * @ring: the engine to test on
793  *
794  */
795 int radeon_vce_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
796 {
797 	struct radeon_fence *fence = NULL;
798 	int r;
799 
800 	r = radeon_vce_get_create_msg(rdev, ring->idx, 1, NULL);
801 	if (r) {
802 		DRM_ERROR("radeon: failed to get create msg (%d).\n", r);
803 		goto error;
804 	}
805 
806 	r = radeon_vce_get_destroy_msg(rdev, ring->idx, 1, &fence);
807 	if (r) {
808 		DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r);
809 		goto error;
810 	}
811 
812 	r = radeon_fence_wait_timeout(fence, false, usecs_to_jiffies(
813 		RADEON_USEC_IB_TEST_TIMEOUT));
814 	if (r < 0) {
815 		DRM_ERROR("radeon: fence wait failed (%d).\n", r);
816 	} else if (r == 0) {
817 		DRM_ERROR("radeon: fence wait timed out.\n");
818 		r = -ETIMEDOUT;
819 	} else {
820 		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
821 		r = 0;
822 	}
823 error:
824 	radeon_fence_unref(&fence);
825 	return r;
826 }
827