1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 
27 #include <linux/firmware.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_pm.h"
33 #include "amdgpu_vcn.h"
34 #include "soc15d.h"
35 
36 /* Firmware Names */
37 #define FIRMWARE_RAVEN		"amdgpu/raven_vcn.bin"
38 #define FIRMWARE_PICASSO	"amdgpu/picasso_vcn.bin"
39 #define FIRMWARE_RAVEN2		"amdgpu/raven2_vcn.bin"
40 #define FIRMWARE_ARCTURUS 	"amdgpu/arcturus_vcn.bin"
41 #define FIRMWARE_RENOIR 	"amdgpu/renoir_vcn.bin"
42 #define FIRMWARE_NAVI10 	"amdgpu/navi10_vcn.bin"
43 #define FIRMWARE_NAVI14 	"amdgpu/navi14_vcn.bin"
44 #define FIRMWARE_NAVI12 	"amdgpu/navi12_vcn.bin"
45 #define FIRMWARE_SIENNA_CICHLID 	"amdgpu/sienna_cichlid_vcn.bin"
46 #define FIRMWARE_NAVY_FLOUNDER 	"amdgpu/navy_flounder_vcn.bin"
47 
48 MODULE_FIRMWARE(FIRMWARE_RAVEN);
49 MODULE_FIRMWARE(FIRMWARE_PICASSO);
50 MODULE_FIRMWARE(FIRMWARE_RAVEN2);
51 MODULE_FIRMWARE(FIRMWARE_ARCTURUS);
52 MODULE_FIRMWARE(FIRMWARE_RENOIR);
53 MODULE_FIRMWARE(FIRMWARE_NAVI10);
54 MODULE_FIRMWARE(FIRMWARE_NAVI14);
55 MODULE_FIRMWARE(FIRMWARE_NAVI12);
56 MODULE_FIRMWARE(FIRMWARE_SIENNA_CICHLID);
57 MODULE_FIRMWARE(FIRMWARE_NAVY_FLOUNDER);
58 
59 static void amdgpu_vcn_idle_work_handler(struct work_struct *work);
60 
61 int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
62 {
63 	unsigned long bo_size;
64 	const char *fw_name;
65 	const struct common_firmware_header *hdr;
66 	unsigned char fw_check;
67 	int i, r;
68 
69 	INIT_DELAYED_WORK(&adev->vcn.idle_work, amdgpu_vcn_idle_work_handler);
70 	mutex_init(&adev->vcn.vcn_pg_lock);
71 	atomic_set(&adev->vcn.total_submission_cnt, 0);
72 	for (i = 0; i < adev->vcn.num_vcn_inst; i++)
73 		atomic_set(&adev->vcn.inst[i].dpg_enc_submission_cnt, 0);
74 
75 	switch (adev->asic_type) {
76 	case CHIP_RAVEN:
77 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
78 			fw_name = FIRMWARE_RAVEN2;
79 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)
80 			fw_name = FIRMWARE_PICASSO;
81 		else
82 			fw_name = FIRMWARE_RAVEN;
83 		break;
84 	case CHIP_ARCTURUS:
85 		fw_name = FIRMWARE_ARCTURUS;
86 		if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
87 		    (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
88 			adev->vcn.indirect_sram = true;
89 		break;
90 	case CHIP_RENOIR:
91 		fw_name = FIRMWARE_RENOIR;
92 		if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
93 		    (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
94 			adev->vcn.indirect_sram = true;
95 		break;
96 	case CHIP_NAVI10:
97 		fw_name = FIRMWARE_NAVI10;
98 		if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
99 		    (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
100 			adev->vcn.indirect_sram = true;
101 		break;
102 	case CHIP_NAVI14:
103 		fw_name = FIRMWARE_NAVI14;
104 		if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
105 		    (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
106 			adev->vcn.indirect_sram = true;
107 		break;
108 	case CHIP_NAVI12:
109 		fw_name = FIRMWARE_NAVI12;
110 		if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
111 		    (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
112 			adev->vcn.indirect_sram = true;
113 		break;
114 	case CHIP_SIENNA_CICHLID:
115 		fw_name = FIRMWARE_SIENNA_CICHLID;
116 		if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
117 		    (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
118 			adev->vcn.indirect_sram = true;
119 		break;
120 	case CHIP_NAVY_FLOUNDER:
121 		fw_name = FIRMWARE_NAVY_FLOUNDER;
122 		if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
123 		    (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
124 			adev->vcn.indirect_sram = true;
125 		break;
126 	default:
127 		return -EINVAL;
128 	}
129 
130 	r = request_firmware(&adev->vcn.fw, fw_name, adev->dev);
131 	if (r) {
132 		dev_err(adev->dev, "amdgpu_vcn: Can't load firmware \"%s\"\n",
133 			fw_name);
134 		return r;
135 	}
136 
137 	r = amdgpu_ucode_validate(adev->vcn.fw);
138 	if (r) {
139 		dev_err(adev->dev, "amdgpu_vcn: Can't validate firmware \"%s\"\n",
140 			fw_name);
141 		release_firmware(adev->vcn.fw);
142 		adev->vcn.fw = NULL;
143 		return r;
144 	}
145 
146 	hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
147 	adev->vcn.fw_version = le32_to_cpu(hdr->ucode_version);
148 
149 	/* Bit 20-23, it is encode major and non-zero for new naming convention.
150 	 * This field is part of version minor and DRM_DISABLED_FLAG in old naming
151 	 * convention. Since the l:wq!atest version minor is 0x5B and DRM_DISABLED_FLAG
152 	 * is zero in old naming convention, this field is always zero so far.
153 	 * These four bits are used to tell which naming convention is present.
154 	 */
155 	fw_check = (le32_to_cpu(hdr->ucode_version) >> 20) & 0xf;
156 	if (fw_check) {
157 		unsigned int dec_ver, enc_major, enc_minor, vep, fw_rev;
158 
159 		fw_rev = le32_to_cpu(hdr->ucode_version) & 0xfff;
160 		enc_minor = (le32_to_cpu(hdr->ucode_version) >> 12) & 0xff;
161 		enc_major = fw_check;
162 		dec_ver = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xf;
163 		vep = (le32_to_cpu(hdr->ucode_version) >> 28) & 0xf;
164 		DRM_INFO("Found VCN firmware Version ENC: %hu.%hu DEC: %hu VEP: %hu Revision: %hu\n",
165 			enc_major, enc_minor, dec_ver, vep, fw_rev);
166 	} else {
167 		unsigned int version_major, version_minor, family_id;
168 
169 		family_id = le32_to_cpu(hdr->ucode_version) & 0xff;
170 		version_major = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xff;
171 		version_minor = (le32_to_cpu(hdr->ucode_version) >> 8) & 0xff;
172 		DRM_INFO("Found VCN firmware Version: %hu.%hu Family ID: %hu\n",
173 			version_major, version_minor, family_id);
174 	}
175 
176 	bo_size = AMDGPU_VCN_STACK_SIZE + AMDGPU_VCN_CONTEXT_SIZE;
177 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
178 		bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
179 	bo_size += AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared));
180 
181 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
182 		if (adev->vcn.harvest_config & (1 << i))
183 			continue;
184 
185 		r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE,
186 						AMDGPU_GEM_DOMAIN_VRAM, &adev->vcn.inst[i].vcpu_bo,
187 						&adev->vcn.inst[i].gpu_addr, &adev->vcn.inst[i].cpu_addr);
188 		if (r) {
189 			dev_err(adev->dev, "(%d) failed to allocate vcn bo\n", r);
190 			return r;
191 		}
192 
193 		adev->vcn.inst[i].fw_shared_cpu_addr = adev->vcn.inst[i].cpu_addr +
194 				bo_size - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared));
195 		adev->vcn.inst[i].fw_shared_gpu_addr = adev->vcn.inst[i].gpu_addr +
196 				bo_size - AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared));
197 
198 		if (adev->vcn.indirect_sram) {
199 			r = amdgpu_bo_create_kernel(adev, 64 * 2 * 4, PAGE_SIZE,
200 					AMDGPU_GEM_DOMAIN_VRAM, &adev->vcn.inst[i].dpg_sram_bo,
201 					&adev->vcn.inst[i].dpg_sram_gpu_addr, &adev->vcn.inst[i].dpg_sram_cpu_addr);
202 			if (r) {
203 				dev_err(adev->dev, "VCN %d (%d) failed to allocate DPG bo\n", i, r);
204 				return r;
205 			}
206 		}
207 	}
208 
209 	return 0;
210 }
211 
212 int amdgpu_vcn_sw_fini(struct amdgpu_device *adev)
213 {
214 	int i, j;
215 
216 	cancel_delayed_work_sync(&adev->vcn.idle_work);
217 
218 	for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
219 		if (adev->vcn.harvest_config & (1 << j))
220 			continue;
221 
222 		if (adev->vcn.indirect_sram) {
223 			amdgpu_bo_free_kernel(&adev->vcn.inst[j].dpg_sram_bo,
224 						  &adev->vcn.inst[j].dpg_sram_gpu_addr,
225 						  (void **)&adev->vcn.inst[j].dpg_sram_cpu_addr);
226 		}
227 		kvfree(adev->vcn.inst[j].saved_bo);
228 
229 		amdgpu_bo_free_kernel(&adev->vcn.inst[j].vcpu_bo,
230 					  &adev->vcn.inst[j].gpu_addr,
231 					  (void **)&adev->vcn.inst[j].cpu_addr);
232 
233 		amdgpu_ring_fini(&adev->vcn.inst[j].ring_dec);
234 
235 		for (i = 0; i < adev->vcn.num_enc_rings; ++i)
236 			amdgpu_ring_fini(&adev->vcn.inst[j].ring_enc[i]);
237 	}
238 
239 	release_firmware(adev->vcn.fw);
240 	mutex_destroy(&adev->vcn.vcn_pg_lock);
241 
242 	return 0;
243 }
244 
245 int amdgpu_vcn_suspend(struct amdgpu_device *adev)
246 {
247 	unsigned size;
248 	void *ptr;
249 	int i;
250 
251 	cancel_delayed_work_sync(&adev->vcn.idle_work);
252 
253 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
254 		if (adev->vcn.harvest_config & (1 << i))
255 			continue;
256 		if (adev->vcn.inst[i].vcpu_bo == NULL)
257 			return 0;
258 
259 		size = amdgpu_bo_size(adev->vcn.inst[i].vcpu_bo);
260 		ptr = adev->vcn.inst[i].cpu_addr;
261 
262 		adev->vcn.inst[i].saved_bo = kvmalloc(size, GFP_KERNEL);
263 		if (!adev->vcn.inst[i].saved_bo)
264 			return -ENOMEM;
265 
266 		memcpy_fromio(adev->vcn.inst[i].saved_bo, ptr, size);
267 	}
268 	return 0;
269 }
270 
271 int amdgpu_vcn_resume(struct amdgpu_device *adev)
272 {
273 	unsigned size;
274 	void *ptr;
275 	int i;
276 
277 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
278 		if (adev->vcn.harvest_config & (1 << i))
279 			continue;
280 		if (adev->vcn.inst[i].vcpu_bo == NULL)
281 			return -EINVAL;
282 
283 		size = amdgpu_bo_size(adev->vcn.inst[i].vcpu_bo);
284 		ptr = adev->vcn.inst[i].cpu_addr;
285 
286 		if (adev->vcn.inst[i].saved_bo != NULL) {
287 			memcpy_toio(ptr, adev->vcn.inst[i].saved_bo, size);
288 			kvfree(adev->vcn.inst[i].saved_bo);
289 			adev->vcn.inst[i].saved_bo = NULL;
290 		} else {
291 			const struct common_firmware_header *hdr;
292 			unsigned offset;
293 
294 			hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
295 			if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
296 				offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
297 				memcpy_toio(adev->vcn.inst[i].cpu_addr, adev->vcn.fw->data + offset,
298 					    le32_to_cpu(hdr->ucode_size_bytes));
299 				size -= le32_to_cpu(hdr->ucode_size_bytes);
300 				ptr += le32_to_cpu(hdr->ucode_size_bytes);
301 			}
302 			memset_io(ptr, 0, size);
303 		}
304 	}
305 	return 0;
306 }
307 
308 static void amdgpu_vcn_idle_work_handler(struct work_struct *work)
309 {
310 	struct amdgpu_device *adev =
311 		container_of(work, struct amdgpu_device, vcn.idle_work.work);
312 	unsigned int fences = 0, fence[AMDGPU_MAX_VCN_INSTANCES] = {0};
313 	unsigned int i, j;
314 
315 	for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
316 		if (adev->vcn.harvest_config & (1 << j))
317 			continue;
318 
319 		for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
320 			fence[j] += amdgpu_fence_count_emitted(&adev->vcn.inst[j].ring_enc[i]);
321 		}
322 
323 		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)	{
324 			struct dpg_pause_state new_state;
325 
326 			if (fence[j] ||
327 				unlikely(atomic_read(&adev->vcn.inst[j].dpg_enc_submission_cnt)))
328 				new_state.fw_based = VCN_DPG_STATE__PAUSE;
329 			else
330 				new_state.fw_based = VCN_DPG_STATE__UNPAUSE;
331 
332 			adev->vcn.pause_dpg_mode(adev, j, &new_state);
333 		}
334 
335 		fence[j] += amdgpu_fence_count_emitted(&adev->vcn.inst[j].ring_dec);
336 		fences += fence[j];
337 	}
338 
339 	if (!fences && !atomic_read(&adev->vcn.total_submission_cnt)) {
340 		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
341 		       AMD_PG_STATE_GATE);
342 	} else {
343 		schedule_delayed_work(&adev->vcn.idle_work, VCN_IDLE_TIMEOUT);
344 	}
345 }
346 
347 void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring)
348 {
349 	struct amdgpu_device *adev = ring->adev;
350 
351 	atomic_inc(&adev->vcn.total_submission_cnt);
352 	cancel_delayed_work_sync(&adev->vcn.idle_work);
353 
354 	mutex_lock(&adev->vcn.vcn_pg_lock);
355 	amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
356 	       AMD_PG_STATE_UNGATE);
357 
358 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)	{
359 		struct dpg_pause_state new_state;
360 
361 		if (ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC) {
362 			atomic_inc(&adev->vcn.inst[ring->me].dpg_enc_submission_cnt);
363 			new_state.fw_based = VCN_DPG_STATE__PAUSE;
364 		} else {
365 			unsigned int fences = 0;
366 			unsigned int i;
367 
368 			for (i = 0; i < adev->vcn.num_enc_rings; ++i)
369 				fences += amdgpu_fence_count_emitted(&adev->vcn.inst[ring->me].ring_enc[i]);
370 
371 			if (fences || atomic_read(&adev->vcn.inst[ring->me].dpg_enc_submission_cnt))
372 				new_state.fw_based = VCN_DPG_STATE__PAUSE;
373 			else
374 				new_state.fw_based = VCN_DPG_STATE__UNPAUSE;
375 		}
376 
377 		adev->vcn.pause_dpg_mode(adev, ring->me, &new_state);
378 	}
379 	mutex_unlock(&adev->vcn.vcn_pg_lock);
380 }
381 
382 void amdgpu_vcn_ring_end_use(struct amdgpu_ring *ring)
383 {
384 	if (ring->adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG &&
385 		ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC)
386 		atomic_dec(&ring->adev->vcn.inst[ring->me].dpg_enc_submission_cnt);
387 
388 	atomic_dec(&ring->adev->vcn.total_submission_cnt);
389 
390 	schedule_delayed_work(&ring->adev->vcn.idle_work, VCN_IDLE_TIMEOUT);
391 }
392 
393 int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring)
394 {
395 	struct amdgpu_device *adev = ring->adev;
396 	uint32_t tmp = 0;
397 	unsigned i;
398 	int r;
399 
400 	/* VCN in SRIOV does not support direct register read/write */
401 	if (amdgpu_sriov_vf(adev))
402 		return 0;
403 
404 	WREG32(adev->vcn.inst[ring->me].external.scratch9, 0xCAFEDEAD);
405 	r = amdgpu_ring_alloc(ring, 3);
406 	if (r)
407 		return r;
408 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.scratch9, 0));
409 	amdgpu_ring_write(ring, 0xDEADBEEF);
410 	amdgpu_ring_commit(ring);
411 	for (i = 0; i < adev->usec_timeout; i++) {
412 		tmp = RREG32(adev->vcn.inst[ring->me].external.scratch9);
413 		if (tmp == 0xDEADBEEF)
414 			break;
415 		udelay(1);
416 	}
417 
418 	if (i >= adev->usec_timeout)
419 		r = -ETIMEDOUT;
420 
421 	return r;
422 }
423 
424 static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring,
425 				   struct amdgpu_bo *bo,
426 				   struct dma_fence **fence)
427 {
428 	struct amdgpu_device *adev = ring->adev;
429 	struct dma_fence *f = NULL;
430 	struct amdgpu_job *job;
431 	struct amdgpu_ib *ib;
432 	uint64_t addr;
433 	int i, r;
434 
435 	r = amdgpu_job_alloc_with_ib(adev, 64,
436 					AMDGPU_IB_POOL_DIRECT, &job);
437 	if (r)
438 		goto err;
439 
440 	ib = &job->ibs[0];
441 	addr = amdgpu_bo_gpu_offset(bo);
442 	ib->ptr[0] = PACKET0(adev->vcn.internal.data0, 0);
443 	ib->ptr[1] = addr;
444 	ib->ptr[2] = PACKET0(adev->vcn.internal.data1, 0);
445 	ib->ptr[3] = addr >> 32;
446 	ib->ptr[4] = PACKET0(adev->vcn.internal.cmd, 0);
447 	ib->ptr[5] = 0;
448 	for (i = 6; i < 16; i += 2) {
449 		ib->ptr[i] = PACKET0(adev->vcn.internal.nop, 0);
450 		ib->ptr[i+1] = 0;
451 	}
452 	ib->length_dw = 16;
453 
454 	r = amdgpu_job_submit_direct(job, ring, &f);
455 	if (r)
456 		goto err_free;
457 
458 	amdgpu_bo_fence(bo, f, false);
459 	amdgpu_bo_unreserve(bo);
460 	amdgpu_bo_unref(&bo);
461 
462 	if (fence)
463 		*fence = dma_fence_get(f);
464 	dma_fence_put(f);
465 
466 	return 0;
467 
468 err_free:
469 	amdgpu_job_free(job);
470 
471 err:
472 	amdgpu_bo_unreserve(bo);
473 	amdgpu_bo_unref(&bo);
474 	return r;
475 }
476 
477 static int amdgpu_vcn_dec_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
478 			      struct dma_fence **fence)
479 {
480 	struct amdgpu_device *adev = ring->adev;
481 	struct amdgpu_bo *bo = NULL;
482 	uint32_t *msg;
483 	int r, i;
484 
485 	r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE,
486 				      AMDGPU_GEM_DOMAIN_VRAM,
487 				      &bo, NULL, (void **)&msg);
488 	if (r)
489 		return r;
490 
491 	msg[0] = cpu_to_le32(0x00000028);
492 	msg[1] = cpu_to_le32(0x00000038);
493 	msg[2] = cpu_to_le32(0x00000001);
494 	msg[3] = cpu_to_le32(0x00000000);
495 	msg[4] = cpu_to_le32(handle);
496 	msg[5] = cpu_to_le32(0x00000000);
497 	msg[6] = cpu_to_le32(0x00000001);
498 	msg[7] = cpu_to_le32(0x00000028);
499 	msg[8] = cpu_to_le32(0x00000010);
500 	msg[9] = cpu_to_le32(0x00000000);
501 	msg[10] = cpu_to_le32(0x00000007);
502 	msg[11] = cpu_to_le32(0x00000000);
503 	msg[12] = cpu_to_le32(0x00000780);
504 	msg[13] = cpu_to_le32(0x00000440);
505 	for (i = 14; i < 1024; ++i)
506 		msg[i] = cpu_to_le32(0x0);
507 
508 	return amdgpu_vcn_dec_send_msg(ring, bo, fence);
509 }
510 
511 static int amdgpu_vcn_dec_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
512 			       struct dma_fence **fence)
513 {
514 	struct amdgpu_device *adev = ring->adev;
515 	struct amdgpu_bo *bo = NULL;
516 	uint32_t *msg;
517 	int r, i;
518 
519 	r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE,
520 				      AMDGPU_GEM_DOMAIN_VRAM,
521 				      &bo, NULL, (void **)&msg);
522 	if (r)
523 		return r;
524 
525 	msg[0] = cpu_to_le32(0x00000028);
526 	msg[1] = cpu_to_le32(0x00000018);
527 	msg[2] = cpu_to_le32(0x00000000);
528 	msg[3] = cpu_to_le32(0x00000002);
529 	msg[4] = cpu_to_le32(handle);
530 	msg[5] = cpu_to_le32(0x00000000);
531 	for (i = 6; i < 1024; ++i)
532 		msg[i] = cpu_to_le32(0x0);
533 
534 	return amdgpu_vcn_dec_send_msg(ring, bo, fence);
535 }
536 
537 int amdgpu_vcn_dec_ring_test_ib(struct amdgpu_ring *ring, long timeout)
538 {
539 	struct dma_fence *fence;
540 	long r;
541 
542 	r = amdgpu_vcn_dec_get_create_msg(ring, 1, NULL);
543 	if (r)
544 		goto error;
545 
546 	r = amdgpu_vcn_dec_get_destroy_msg(ring, 1, &fence);
547 	if (r)
548 		goto error;
549 
550 	r = dma_fence_wait_timeout(fence, false, timeout);
551 	if (r == 0)
552 		r = -ETIMEDOUT;
553 	else if (r > 0)
554 		r = 0;
555 
556 	dma_fence_put(fence);
557 error:
558 	return r;
559 }
560 
561 int amdgpu_vcn_enc_ring_test_ring(struct amdgpu_ring *ring)
562 {
563 	struct amdgpu_device *adev = ring->adev;
564 	uint32_t rptr;
565 	unsigned i;
566 	int r;
567 
568 	if (amdgpu_sriov_vf(adev))
569 		return 0;
570 
571 	r = amdgpu_ring_alloc(ring, 16);
572 	if (r)
573 		return r;
574 
575 	rptr = amdgpu_ring_get_rptr(ring);
576 
577 	amdgpu_ring_write(ring, VCN_ENC_CMD_END);
578 	amdgpu_ring_commit(ring);
579 
580 	for (i = 0; i < adev->usec_timeout; i++) {
581 		if (amdgpu_ring_get_rptr(ring) != rptr)
582 			break;
583 		udelay(1);
584 	}
585 
586 	if (i >= adev->usec_timeout)
587 		r = -ETIMEDOUT;
588 
589 	return r;
590 }
591 
592 static int amdgpu_vcn_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
593 					 struct amdgpu_bo *bo,
594 					 struct dma_fence **fence)
595 {
596 	const unsigned ib_size_dw = 16;
597 	struct amdgpu_job *job;
598 	struct amdgpu_ib *ib;
599 	struct dma_fence *f = NULL;
600 	uint64_t addr;
601 	int i, r;
602 
603 	r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4,
604 					AMDGPU_IB_POOL_DIRECT, &job);
605 	if (r)
606 		return r;
607 
608 	ib = &job->ibs[0];
609 	addr = amdgpu_bo_gpu_offset(bo);
610 
611 	ib->length_dw = 0;
612 	ib->ptr[ib->length_dw++] = 0x00000018;
613 	ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
614 	ib->ptr[ib->length_dw++] = handle;
615 	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
616 	ib->ptr[ib->length_dw++] = addr;
617 	ib->ptr[ib->length_dw++] = 0x0000000b;
618 
619 	ib->ptr[ib->length_dw++] = 0x00000014;
620 	ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
621 	ib->ptr[ib->length_dw++] = 0x0000001c;
622 	ib->ptr[ib->length_dw++] = 0x00000000;
623 	ib->ptr[ib->length_dw++] = 0x00000000;
624 
625 	ib->ptr[ib->length_dw++] = 0x00000008;
626 	ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */
627 
628 	for (i = ib->length_dw; i < ib_size_dw; ++i)
629 		ib->ptr[i] = 0x0;
630 
631 	r = amdgpu_job_submit_direct(job, ring, &f);
632 	if (r)
633 		goto err;
634 
635 	if (fence)
636 		*fence = dma_fence_get(f);
637 	dma_fence_put(f);
638 
639 	return 0;
640 
641 err:
642 	amdgpu_job_free(job);
643 	return r;
644 }
645 
646 static int amdgpu_vcn_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
647 					  struct amdgpu_bo *bo,
648 					  struct dma_fence **fence)
649 {
650 	const unsigned ib_size_dw = 16;
651 	struct amdgpu_job *job;
652 	struct amdgpu_ib *ib;
653 	struct dma_fence *f = NULL;
654 	uint64_t addr;
655 	int i, r;
656 
657 	r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4,
658 					AMDGPU_IB_POOL_DIRECT, &job);
659 	if (r)
660 		return r;
661 
662 	ib = &job->ibs[0];
663 	addr = amdgpu_bo_gpu_offset(bo);
664 
665 	ib->length_dw = 0;
666 	ib->ptr[ib->length_dw++] = 0x00000018;
667 	ib->ptr[ib->length_dw++] = 0x00000001;
668 	ib->ptr[ib->length_dw++] = handle;
669 	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
670 	ib->ptr[ib->length_dw++] = addr;
671 	ib->ptr[ib->length_dw++] = 0x0000000b;
672 
673 	ib->ptr[ib->length_dw++] = 0x00000014;
674 	ib->ptr[ib->length_dw++] = 0x00000002;
675 	ib->ptr[ib->length_dw++] = 0x0000001c;
676 	ib->ptr[ib->length_dw++] = 0x00000000;
677 	ib->ptr[ib->length_dw++] = 0x00000000;
678 
679 	ib->ptr[ib->length_dw++] = 0x00000008;
680 	ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */
681 
682 	for (i = ib->length_dw; i < ib_size_dw; ++i)
683 		ib->ptr[i] = 0x0;
684 
685 	r = amdgpu_job_submit_direct(job, ring, &f);
686 	if (r)
687 		goto err;
688 
689 	if (fence)
690 		*fence = dma_fence_get(f);
691 	dma_fence_put(f);
692 
693 	return 0;
694 
695 err:
696 	amdgpu_job_free(job);
697 	return r;
698 }
699 
700 int amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
701 {
702 	struct dma_fence *fence = NULL;
703 	struct amdgpu_bo *bo = NULL;
704 	long r;
705 
706 	r = amdgpu_bo_create_reserved(ring->adev, 128 * 1024, PAGE_SIZE,
707 				      AMDGPU_GEM_DOMAIN_VRAM,
708 				      &bo, NULL, NULL);
709 	if (r)
710 		return r;
711 
712 	r = amdgpu_vcn_enc_get_create_msg(ring, 1, bo, NULL);
713 	if (r)
714 		goto error;
715 
716 	r = amdgpu_vcn_enc_get_destroy_msg(ring, 1, bo, &fence);
717 	if (r)
718 		goto error;
719 
720 	r = dma_fence_wait_timeout(fence, false, timeout);
721 	if (r == 0)
722 		r = -ETIMEDOUT;
723 	else if (r > 0)
724 		r = 0;
725 
726 error:
727 	dma_fence_put(fence);
728 	amdgpu_bo_unreserve(bo);
729 	amdgpu_bo_unref(&bo);
730 	return r;
731 }
732