1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include "amdgpu.h"
25 #include "amdgpu_jpeg.h"
26 #include "amdgpu_pm.h"
27 #include "soc15.h"
28 #include "soc15d.h"
29 #include "jpeg_v2_0.h"
30 
31 #include "vcn/vcn_2_0_0_offset.h"
32 #include "vcn/vcn_2_0_0_sh_mask.h"
33 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
34 
35 static void jpeg_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev);
36 static void jpeg_v2_0_set_irq_funcs(struct amdgpu_device *adev);
37 static int jpeg_v2_0_set_powergating_state(void *handle,
38 				enum amd_powergating_state state);
39 
40 /**
41  * jpeg_v2_0_early_init - set function pointers
42  *
43  * @handle: amdgpu_device pointer
44  *
45  * Set ring and irq function pointers
46  */
47 static int jpeg_v2_0_early_init(void *handle)
48 {
49 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
50 
51 	adev->jpeg.num_jpeg_inst = 1;
52 
53 	jpeg_v2_0_set_dec_ring_funcs(adev);
54 	jpeg_v2_0_set_irq_funcs(adev);
55 
56 	return 0;
57 }
58 
59 /**
60  * jpeg_v2_0_sw_init - sw init for JPEG block
61  *
62  * @handle: amdgpu_device pointer
63  *
64  * Load firmware and sw initialization
65  */
66 static int jpeg_v2_0_sw_init(void *handle)
67 {
68 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
69 	struct amdgpu_ring *ring;
70 	int r;
71 
72 	/* JPEG TRAP */
73 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
74 		VCN_2_0__SRCID__JPEG_DECODE, &adev->jpeg.inst->irq);
75 	if (r)
76 		return r;
77 
78 	r = amdgpu_jpeg_sw_init(adev);
79 	if (r)
80 		return r;
81 
82 	r = amdgpu_jpeg_resume(adev);
83 	if (r)
84 		return r;
85 
86 	ring = &adev->jpeg.inst->ring_dec;
87 	ring->use_doorbell = true;
88 	ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1;
89 	sprintf(ring->name, "jpeg_dec");
90 	r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq,
91 			     0, AMDGPU_RING_PRIO_DEFAULT, NULL);
92 	if (r)
93 		return r;
94 
95 	adev->jpeg.internal.jpeg_pitch = mmUVD_JPEG_PITCH_INTERNAL_OFFSET;
96 	adev->jpeg.inst->external.jpeg_pitch = SOC15_REG_OFFSET(JPEG, 0, mmUVD_JPEG_PITCH);
97 
98 	return 0;
99 }
100 
101 /**
102  * jpeg_v2_0_sw_fini - sw fini for JPEG block
103  *
104  * @handle: amdgpu_device pointer
105  *
106  * JPEG suspend and free up sw allocation
107  */
108 static int jpeg_v2_0_sw_fini(void *handle)
109 {
110 	int r;
111 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
112 
113 	r = amdgpu_jpeg_suspend(adev);
114 	if (r)
115 		return r;
116 
117 	r = amdgpu_jpeg_sw_fini(adev);
118 
119 	return r;
120 }
121 
122 /**
123  * jpeg_v2_0_hw_init - start and test JPEG block
124  *
125  * @handle: amdgpu_device pointer
126  *
127  */
128 static int jpeg_v2_0_hw_init(void *handle)
129 {
130 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
131 	struct amdgpu_ring *ring = &adev->jpeg.inst->ring_dec;
132 	int r;
133 
134 	adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
135 		(adev->doorbell_index.vcn.vcn_ring0_1 << 1), 0);
136 
137 	r = amdgpu_ring_test_helper(ring);
138 	if (!r)
139 		DRM_INFO("JPEG decode initialized successfully.\n");
140 
141 	return r;
142 }
143 
144 /**
145  * jpeg_v2_0_hw_fini - stop the hardware block
146  *
147  * @handle: amdgpu_device pointer
148  *
149  * Stop the JPEG block, mark ring as not ready any more
150  */
151 static int jpeg_v2_0_hw_fini(void *handle)
152 {
153 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
154 
155 	cancel_delayed_work_sync(&adev->vcn.idle_work);
156 
157 	if (adev->jpeg.cur_state != AMD_PG_STATE_GATE &&
158 	      RREG32_SOC15(JPEG, 0, mmUVD_JRBC_STATUS))
159 		jpeg_v2_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
160 
161 	return 0;
162 }
163 
164 /**
165  * jpeg_v2_0_suspend - suspend JPEG block
166  *
167  * @handle: amdgpu_device pointer
168  *
169  * HW fini and suspend JPEG block
170  */
171 static int jpeg_v2_0_suspend(void *handle)
172 {
173 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
174 	int r;
175 
176 	r = jpeg_v2_0_hw_fini(adev);
177 	if (r)
178 		return r;
179 
180 	r = amdgpu_jpeg_suspend(adev);
181 
182 	return r;
183 }
184 
185 /**
186  * jpeg_v2_0_resume - resume JPEG block
187  *
188  * @handle: amdgpu_device pointer
189  *
190  * Resume firmware and hw init JPEG block
191  */
192 static int jpeg_v2_0_resume(void *handle)
193 {
194 	int r;
195 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
196 
197 	r = amdgpu_jpeg_resume(adev);
198 	if (r)
199 		return r;
200 
201 	r = jpeg_v2_0_hw_init(adev);
202 
203 	return r;
204 }
205 
206 static int jpeg_v2_0_disable_power_gating(struct amdgpu_device *adev)
207 {
208 	uint32_t data;
209 	int r = 0;
210 
211 	if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
212 		data = 1 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT;
213 		WREG32(SOC15_REG_OFFSET(JPEG, 0, mmUVD_PGFSM_CONFIG), data);
214 
215 		r = SOC15_WAIT_ON_RREG(JPEG, 0,
216 			mmUVD_PGFSM_STATUS, UVD_PGFSM_STATUS_UVDJ_PWR_ON,
217 			UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
218 
219 		if (r) {
220 			DRM_ERROR("amdgpu: JPEG disable power gating failed\n");
221 			return r;
222 		}
223 	}
224 
225 	/* Removing the anti hang mechanism to indicate the UVDJ tile is ON */
226 	data = RREG32(SOC15_REG_OFFSET(JPEG, 0, mmUVD_JPEG_POWER_STATUS)) & ~0x1;
227 	WREG32(SOC15_REG_OFFSET(JPEG, 0, mmUVD_JPEG_POWER_STATUS), data);
228 
229 	return 0;
230 }
231 
232 static int jpeg_v2_0_enable_power_gating(struct amdgpu_device *adev)
233 {
234 	if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
235 		uint32_t data;
236 		int r = 0;
237 
238 		data = RREG32(SOC15_REG_OFFSET(JPEG, 0, mmUVD_JPEG_POWER_STATUS));
239 		data &= ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK;
240 		data |=  0x1; //UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_TILES_OFF;
241 		WREG32(SOC15_REG_OFFSET(JPEG, 0, mmUVD_JPEG_POWER_STATUS), data);
242 
243 		data = 2 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT;
244 		WREG32(SOC15_REG_OFFSET(JPEG, 0, mmUVD_PGFSM_CONFIG), data);
245 
246 		r = SOC15_WAIT_ON_RREG(JPEG, 0, mmUVD_PGFSM_STATUS,
247 			(2 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT),
248 			UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
249 
250 		if (r) {
251 			DRM_ERROR("amdgpu: JPEG enable power gating failed\n");
252 			return r;
253 		}
254 	}
255 
256 	return 0;
257 }
258 
259 static void jpeg_v2_0_disable_clock_gating(struct amdgpu_device *adev)
260 {
261 	uint32_t data;
262 
263 	data = RREG32_SOC15(JPEG, 0, mmJPEG_CGC_CTRL);
264 	if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG)
265 		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
266 	else
267 		data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
268 
269 	data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
270 	data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
271 	WREG32_SOC15(JPEG, 0, mmJPEG_CGC_CTRL, data);
272 
273 	data = RREG32_SOC15(JPEG, 0, mmJPEG_CGC_GATE);
274 	data &= ~(JPEG_CGC_GATE__JPEG_DEC_MASK
275 		| JPEG_CGC_GATE__JPEG2_DEC_MASK
276 		| JPEG_CGC_GATE__JPEG_ENC_MASK
277 		| JPEG_CGC_GATE__JMCIF_MASK
278 		| JPEG_CGC_GATE__JRBBM_MASK);
279 	WREG32_SOC15(JPEG, 0, mmJPEG_CGC_GATE, data);
280 }
281 
282 static void jpeg_v2_0_enable_clock_gating(struct amdgpu_device *adev)
283 {
284 	uint32_t data;
285 
286 	data = RREG32_SOC15(JPEG, 0, mmJPEG_CGC_CTRL);
287 	if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG)
288 		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
289 	else
290 		data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
291 
292 	data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
293 	data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
294 	WREG32_SOC15(JPEG, 0, mmJPEG_CGC_CTRL, data);
295 
296 	data = RREG32_SOC15(JPEG, 0, mmJPEG_CGC_GATE);
297 	data |= (JPEG_CGC_GATE__JPEG_DEC_MASK
298 		|JPEG_CGC_GATE__JPEG2_DEC_MASK
299 		|JPEG_CGC_GATE__JPEG_ENC_MASK
300 		|JPEG_CGC_GATE__JMCIF_MASK
301 		|JPEG_CGC_GATE__JRBBM_MASK);
302 	WREG32_SOC15(JPEG, 0, mmJPEG_CGC_GATE, data);
303 }
304 
305 /**
306  * jpeg_v2_0_start - start JPEG block
307  *
308  * @adev: amdgpu_device pointer
309  *
310  * Setup and start the JPEG block
311  */
312 static int jpeg_v2_0_start(struct amdgpu_device *adev)
313 {
314 	struct amdgpu_ring *ring = &adev->jpeg.inst->ring_dec;
315 	int r;
316 
317 	if (adev->pm.dpm_enabled)
318 		amdgpu_dpm_enable_jpeg(adev, true);
319 
320 	/* disable power gating */
321 	r = jpeg_v2_0_disable_power_gating(adev);
322 	if (r)
323 		return r;
324 
325 	/* JPEG disable CGC */
326 	jpeg_v2_0_disable_clock_gating(adev);
327 
328 	WREG32_SOC15(JPEG, 0, mmJPEG_DEC_GFX10_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
329 
330 	/* enable JMI channel */
331 	WREG32_P(SOC15_REG_OFFSET(JPEG, 0, mmUVD_JMI_CNTL), 0,
332 		~UVD_JMI_CNTL__SOFT_RESET_MASK);
333 
334 	/* enable System Interrupt for JRBC */
335 	WREG32_P(SOC15_REG_OFFSET(JPEG, 0, mmJPEG_SYS_INT_EN),
336 		JPEG_SYS_INT_EN__DJRBC_MASK,
337 		~JPEG_SYS_INT_EN__DJRBC_MASK);
338 
339 	WREG32_SOC15(JPEG, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
340 	WREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
341 	WREG32_SOC15(JPEG, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
342 		lower_32_bits(ring->gpu_addr));
343 	WREG32_SOC15(JPEG, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
344 		upper_32_bits(ring->gpu_addr));
345 	WREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_RPTR, 0);
346 	WREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_WPTR, 0);
347 	WREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_CNTL, 0x00000002L);
348 	WREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_SIZE, ring->ring_size / 4);
349 	ring->wptr = RREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_WPTR);
350 
351 	return 0;
352 }
353 
354 /**
355  * jpeg_v2_0_stop - stop JPEG block
356  *
357  * @adev: amdgpu_device pointer
358  *
359  * stop the JPEG block
360  */
361 static int jpeg_v2_0_stop(struct amdgpu_device *adev)
362 {
363 	int r;
364 
365 	/* reset JMI */
366 	WREG32_P(SOC15_REG_OFFSET(JPEG, 0, mmUVD_JMI_CNTL),
367 		UVD_JMI_CNTL__SOFT_RESET_MASK,
368 		~UVD_JMI_CNTL__SOFT_RESET_MASK);
369 
370 	/* enable JPEG CGC */
371 	jpeg_v2_0_enable_clock_gating(adev);
372 
373 	/* enable power gating */
374 	r = jpeg_v2_0_enable_power_gating(adev);
375 	if (r)
376 		return r;
377 
378 	if (adev->pm.dpm_enabled)
379 		amdgpu_dpm_enable_jpeg(adev, false);
380 
381 	return 0;
382 }
383 
384 /**
385  * jpeg_v2_0_dec_ring_get_rptr - get read pointer
386  *
387  * @ring: amdgpu_ring pointer
388  *
389  * Returns the current hardware read pointer
390  */
391 static uint64_t jpeg_v2_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
392 {
393 	struct amdgpu_device *adev = ring->adev;
394 
395 	return RREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_RPTR);
396 }
397 
398 /**
399  * jpeg_v2_0_dec_ring_get_wptr - get write pointer
400  *
401  * @ring: amdgpu_ring pointer
402  *
403  * Returns the current hardware write pointer
404  */
405 static uint64_t jpeg_v2_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
406 {
407 	struct amdgpu_device *adev = ring->adev;
408 
409 	if (ring->use_doorbell)
410 		return adev->wb.wb[ring->wptr_offs];
411 	else
412 		return RREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_WPTR);
413 }
414 
415 /**
416  * jpeg_v2_0_dec_ring_set_wptr - set write pointer
417  *
418  * @ring: amdgpu_ring pointer
419  *
420  * Commits the write pointer to the hardware
421  */
422 static void jpeg_v2_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
423 {
424 	struct amdgpu_device *adev = ring->adev;
425 
426 	if (ring->use_doorbell) {
427 		adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
428 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
429 	} else {
430 		WREG32_SOC15(JPEG, 0, mmUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
431 	}
432 }
433 
434 /**
435  * jpeg_v2_0_dec_ring_insert_start - insert a start command
436  *
437  * @ring: amdgpu_ring pointer
438  *
439  * Write a start command to the ring.
440  */
441 void jpeg_v2_0_dec_ring_insert_start(struct amdgpu_ring *ring)
442 {
443 	amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
444 		0, 0, PACKETJ_TYPE0));
445 	amdgpu_ring_write(ring, 0x68e04);
446 
447 	amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
448 		0, 0, PACKETJ_TYPE0));
449 	amdgpu_ring_write(ring, 0x80010000);
450 }
451 
452 /**
453  * jpeg_v2_0_dec_ring_insert_end - insert a end command
454  *
455  * @ring: amdgpu_ring pointer
456  *
457  * Write a end command to the ring.
458  */
459 void jpeg_v2_0_dec_ring_insert_end(struct amdgpu_ring *ring)
460 {
461 	amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
462 		0, 0, PACKETJ_TYPE0));
463 	amdgpu_ring_write(ring, 0x68e04);
464 
465 	amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
466 		0, 0, PACKETJ_TYPE0));
467 	amdgpu_ring_write(ring, 0x00010000);
468 }
469 
470 /**
471  * jpeg_v2_0_dec_ring_emit_fence - emit an fence & trap command
472  *
473  * @ring: amdgpu_ring pointer
474  * @addr: address
475  * @seq: sequence number
476  * @flags: fence related flags
477  *
478  * Write a fence and a trap command to the ring.
479  */
480 void jpeg_v2_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
481 				unsigned flags)
482 {
483 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
484 
485 	amdgpu_ring_write(ring, PACKETJ(mmUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET,
486 		0, 0, PACKETJ_TYPE0));
487 	amdgpu_ring_write(ring, seq);
488 
489 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET,
490 		0, 0, PACKETJ_TYPE0));
491 	amdgpu_ring_write(ring, seq);
492 
493 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET,
494 		0, 0, PACKETJ_TYPE0));
495 	amdgpu_ring_write(ring, lower_32_bits(addr));
496 
497 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET,
498 		0, 0, PACKETJ_TYPE0));
499 	amdgpu_ring_write(ring, upper_32_bits(addr));
500 
501 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
502 		0, 0, PACKETJ_TYPE0));
503 	amdgpu_ring_write(ring, 0x8);
504 
505 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
506 		0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
507 	amdgpu_ring_write(ring, 0);
508 
509 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
510 		0, 0, PACKETJ_TYPE0));
511 	amdgpu_ring_write(ring, 0x3fbc);
512 
513 	amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
514 		0, 0, PACKETJ_TYPE0));
515 	amdgpu_ring_write(ring, 0x1);
516 
517 	amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
518 	amdgpu_ring_write(ring, 0);
519 }
520 
521 /**
522  * jpeg_v2_0_dec_ring_emit_ib - execute indirect buffer
523  *
524  * @ring: amdgpu_ring pointer
525  * @job: job to retrieve vmid from
526  * @ib: indirect buffer to execute
527  * @flags: unused
528  *
529  * Write ring commands to execute the indirect buffer.
530  */
531 void jpeg_v2_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
532 				struct amdgpu_job *job,
533 				struct amdgpu_ib *ib,
534 				uint32_t flags)
535 {
536 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
537 
538 	amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET,
539 		0, 0, PACKETJ_TYPE0));
540 	amdgpu_ring_write(ring, (vmid | (vmid << 4)));
541 
542 	amdgpu_ring_write(ring, PACKETJ(mmUVD_LMI_JPEG_VMID_INTERNAL_OFFSET,
543 		0, 0, PACKETJ_TYPE0));
544 	amdgpu_ring_write(ring, (vmid | (vmid << 4)));
545 
546 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET,
547 		0, 0, PACKETJ_TYPE0));
548 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
549 
550 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET,
551 		0, 0, PACKETJ_TYPE0));
552 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
553 
554 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JRBC_IB_SIZE_INTERNAL_OFFSET,
555 		0, 0, PACKETJ_TYPE0));
556 	amdgpu_ring_write(ring, ib->length_dw);
557 
558 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET,
559 		0, 0, PACKETJ_TYPE0));
560 	amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
561 
562 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET,
563 		0, 0, PACKETJ_TYPE0));
564 	amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
565 
566 	amdgpu_ring_write(ring,	PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
567 	amdgpu_ring_write(ring, 0);
568 
569 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
570 		0, 0, PACKETJ_TYPE0));
571 	amdgpu_ring_write(ring, 0x01400200);
572 
573 	amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
574 		0, 0, PACKETJ_TYPE0));
575 	amdgpu_ring_write(ring, 0x2);
576 
577 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JRBC_STATUS_INTERNAL_OFFSET,
578 		0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
579 	amdgpu_ring_write(ring, 0x2);
580 }
581 
582 void jpeg_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
583 				uint32_t val, uint32_t mask)
584 {
585 	uint32_t reg_offset = (reg << 2);
586 
587 	amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
588 		0, 0, PACKETJ_TYPE0));
589 	amdgpu_ring_write(ring, 0x01400200);
590 
591 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
592 		0, 0, PACKETJ_TYPE0));
593 	amdgpu_ring_write(ring, val);
594 
595 	amdgpu_ring_write(ring, PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
596 		0, 0, PACKETJ_TYPE0));
597 	if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
598 		amdgpu_ring_write(ring, 0);
599 		amdgpu_ring_write(ring,
600 			PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
601 	} else {
602 		amdgpu_ring_write(ring, reg_offset);
603 		amdgpu_ring_write(ring,	PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
604 			0, 0, PACKETJ_TYPE3));
605 	}
606 	amdgpu_ring_write(ring, mask);
607 }
608 
609 void jpeg_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
610 				unsigned vmid, uint64_t pd_addr)
611 {
612 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
613 	uint32_t data0, data1, mask;
614 
615 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
616 
617 	/* wait for register write */
618 	data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
619 	data1 = lower_32_bits(pd_addr);
620 	mask = 0xffffffff;
621 	jpeg_v2_0_dec_ring_emit_reg_wait(ring, data0, data1, mask);
622 }
623 
624 void jpeg_v2_0_dec_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
625 {
626 	uint32_t reg_offset = (reg << 2);
627 
628 	amdgpu_ring_write(ring,	PACKETJ(mmUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
629 		0, 0, PACKETJ_TYPE0));
630 	if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
631 		amdgpu_ring_write(ring, 0);
632 		amdgpu_ring_write(ring,
633 			PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
634 	} else {
635 		amdgpu_ring_write(ring, reg_offset);
636 		amdgpu_ring_write(ring,	PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
637 			0, 0, PACKETJ_TYPE0));
638 	}
639 	amdgpu_ring_write(ring, val);
640 }
641 
642 void jpeg_v2_0_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count)
643 {
644 	int i;
645 
646 	WARN_ON(ring->wptr % 2 || count % 2);
647 
648 	for (i = 0; i < count / 2; i++) {
649 		amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
650 		amdgpu_ring_write(ring, 0);
651 	}
652 }
653 
654 static bool jpeg_v2_0_is_idle(void *handle)
655 {
656 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
657 
658 	return ((RREG32_SOC15(JPEG, 0, mmUVD_JRBC_STATUS) &
659 		UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
660 		UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
661 }
662 
663 static int jpeg_v2_0_wait_for_idle(void *handle)
664 {
665 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
666 	int ret;
667 
668 	ret = SOC15_WAIT_ON_RREG(JPEG, 0, mmUVD_JRBC_STATUS, UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
669 		UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
670 
671 	return ret;
672 }
673 
674 static int jpeg_v2_0_set_clockgating_state(void *handle,
675 					  enum amd_clockgating_state state)
676 {
677 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
678 	bool enable = (state == AMD_CG_STATE_GATE);
679 
680 	if (enable) {
681 		if (!jpeg_v2_0_is_idle(handle))
682 			return -EBUSY;
683 		jpeg_v2_0_enable_clock_gating(adev);
684 	} else {
685 		jpeg_v2_0_disable_clock_gating(adev);
686 	}
687 
688 	return 0;
689 }
690 
691 static int jpeg_v2_0_set_powergating_state(void *handle,
692 					enum amd_powergating_state state)
693 {
694 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
695 	int ret;
696 
697 	if (state == adev->jpeg.cur_state)
698 		return 0;
699 
700 	if (state == AMD_PG_STATE_GATE)
701 		ret = jpeg_v2_0_stop(adev);
702 	else
703 		ret = jpeg_v2_0_start(adev);
704 
705 	if (!ret)
706 		adev->jpeg.cur_state = state;
707 
708 	return ret;
709 }
710 
711 static int jpeg_v2_0_set_interrupt_state(struct amdgpu_device *adev,
712 					struct amdgpu_irq_src *source,
713 					unsigned type,
714 					enum amdgpu_interrupt_state state)
715 {
716 	return 0;
717 }
718 
719 static int jpeg_v2_0_process_interrupt(struct amdgpu_device *adev,
720 				      struct amdgpu_irq_src *source,
721 				      struct amdgpu_iv_entry *entry)
722 {
723 	DRM_DEBUG("IH: JPEG TRAP\n");
724 
725 	switch (entry->src_id) {
726 	case VCN_2_0__SRCID__JPEG_DECODE:
727 		amdgpu_fence_process(&adev->jpeg.inst->ring_dec);
728 		break;
729 	default:
730 		DRM_ERROR("Unhandled interrupt: %d %d\n",
731 			  entry->src_id, entry->src_data[0]);
732 		break;
733 	}
734 
735 	return 0;
736 }
737 
738 static const struct amd_ip_funcs jpeg_v2_0_ip_funcs = {
739 	.name = "jpeg_v2_0",
740 	.early_init = jpeg_v2_0_early_init,
741 	.late_init = NULL,
742 	.sw_init = jpeg_v2_0_sw_init,
743 	.sw_fini = jpeg_v2_0_sw_fini,
744 	.hw_init = jpeg_v2_0_hw_init,
745 	.hw_fini = jpeg_v2_0_hw_fini,
746 	.suspend = jpeg_v2_0_suspend,
747 	.resume = jpeg_v2_0_resume,
748 	.is_idle = jpeg_v2_0_is_idle,
749 	.wait_for_idle = jpeg_v2_0_wait_for_idle,
750 	.check_soft_reset = NULL,
751 	.pre_soft_reset = NULL,
752 	.soft_reset = NULL,
753 	.post_soft_reset = NULL,
754 	.set_clockgating_state = jpeg_v2_0_set_clockgating_state,
755 	.set_powergating_state = jpeg_v2_0_set_powergating_state,
756 };
757 
758 static const struct amdgpu_ring_funcs jpeg_v2_0_dec_ring_vm_funcs = {
759 	.type = AMDGPU_RING_TYPE_VCN_JPEG,
760 	.align_mask = 0xf,
761 	.vmhub = AMDGPU_MMHUB_0,
762 	.get_rptr = jpeg_v2_0_dec_ring_get_rptr,
763 	.get_wptr = jpeg_v2_0_dec_ring_get_wptr,
764 	.set_wptr = jpeg_v2_0_dec_ring_set_wptr,
765 	.emit_frame_size =
766 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
767 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
768 		8 + /* jpeg_v2_0_dec_ring_emit_vm_flush */
769 		18 + 18 + /* jpeg_v2_0_dec_ring_emit_fence x2 vm fence */
770 		8 + 16,
771 	.emit_ib_size = 22, /* jpeg_v2_0_dec_ring_emit_ib */
772 	.emit_ib = jpeg_v2_0_dec_ring_emit_ib,
773 	.emit_fence = jpeg_v2_0_dec_ring_emit_fence,
774 	.emit_vm_flush = jpeg_v2_0_dec_ring_emit_vm_flush,
775 	.test_ring = amdgpu_jpeg_dec_ring_test_ring,
776 	.test_ib = amdgpu_jpeg_dec_ring_test_ib,
777 	.insert_nop = jpeg_v2_0_dec_ring_nop,
778 	.insert_start = jpeg_v2_0_dec_ring_insert_start,
779 	.insert_end = jpeg_v2_0_dec_ring_insert_end,
780 	.pad_ib = amdgpu_ring_generic_pad_ib,
781 	.begin_use = amdgpu_jpeg_ring_begin_use,
782 	.end_use = amdgpu_jpeg_ring_end_use,
783 	.emit_wreg = jpeg_v2_0_dec_ring_emit_wreg,
784 	.emit_reg_wait = jpeg_v2_0_dec_ring_emit_reg_wait,
785 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
786 };
787 
788 static void jpeg_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev)
789 {
790 	adev->jpeg.inst->ring_dec.funcs = &jpeg_v2_0_dec_ring_vm_funcs;
791 	DRM_INFO("JPEG decode is enabled in VM mode\n");
792 }
793 
794 static const struct amdgpu_irq_src_funcs jpeg_v2_0_irq_funcs = {
795 	.set = jpeg_v2_0_set_interrupt_state,
796 	.process = jpeg_v2_0_process_interrupt,
797 };
798 
799 static void jpeg_v2_0_set_irq_funcs(struct amdgpu_device *adev)
800 {
801 	adev->jpeg.inst->irq.num_types = 1;
802 	adev->jpeg.inst->irq.funcs = &jpeg_v2_0_irq_funcs;
803 }
804 
805 const struct amdgpu_ip_block_version jpeg_v2_0_ip_block =
806 {
807 		.type = AMD_IP_BLOCK_TYPE_JPEG,
808 		.major = 2,
809 		.minor = 0,
810 		.rev = 0,
811 		.funcs = &jpeg_v2_0_ip_funcs,
812 };
813