1 /*
2 * Copyright 2022 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include "amdgpu.h"
25 #include "amdgpu_jpeg.h"
26 #include "amdgpu_cs.h"
27 #include "soc15.h"
28 #include "soc15d.h"
29 #include "jpeg_v4_0_3.h"
30 #include "mmsch_v4_0_3.h"
31
32 #include "vcn/vcn_4_0_3_offset.h"
33 #include "vcn/vcn_4_0_3_sh_mask.h"
34 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
35
36 enum jpeg_engin_status {
37 UVD_PGFSM_STATUS__UVDJ_PWR_ON = 0,
38 UVD_PGFSM_STATUS__UVDJ_PWR_OFF = 2,
39 };
40
41 static void jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device *adev);
42 static void jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device *adev);
43 static int jpeg_v4_0_3_set_powergating_state(void *handle,
44 enum amd_powergating_state state);
45 static void jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device *adev);
46 static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring);
47
48 static int amdgpu_ih_srcid_jpeg[] = {
49 VCN_4_0__SRCID__JPEG_DECODE,
50 VCN_4_0__SRCID__JPEG1_DECODE,
51 VCN_4_0__SRCID__JPEG2_DECODE,
52 VCN_4_0__SRCID__JPEG3_DECODE,
53 VCN_4_0__SRCID__JPEG4_DECODE,
54 VCN_4_0__SRCID__JPEG5_DECODE,
55 VCN_4_0__SRCID__JPEG6_DECODE,
56 VCN_4_0__SRCID__JPEG7_DECODE
57 };
58
59 /**
60 * jpeg_v4_0_3_early_init - set function pointers
61 *
62 * @handle: amdgpu_device pointer
63 *
64 * Set ring and irq function pointers
65 */
jpeg_v4_0_3_early_init(void * handle)66 static int jpeg_v4_0_3_early_init(void *handle)
67 {
68 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
69
70 adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS;
71
72 jpeg_v4_0_3_set_dec_ring_funcs(adev);
73 jpeg_v4_0_3_set_irq_funcs(adev);
74 jpeg_v4_0_3_set_ras_funcs(adev);
75
76 return 0;
77 }
78
79 /**
80 * jpeg_v4_0_3_sw_init - sw init for JPEG block
81 *
82 * @handle: amdgpu_device pointer
83 *
84 * Load firmware and sw initialization
85 */
jpeg_v4_0_3_sw_init(void * handle)86 static int jpeg_v4_0_3_sw_init(void *handle)
87 {
88 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
89 struct amdgpu_ring *ring;
90 int i, j, r, jpeg_inst;
91
92 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
93 /* JPEG TRAP */
94 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
95 amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst->irq);
96 if (r)
97 return r;
98 }
99
100 r = amdgpu_jpeg_sw_init(adev);
101 if (r)
102 return r;
103
104 r = amdgpu_jpeg_resume(adev);
105 if (r)
106 return r;
107
108 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
109 jpeg_inst = GET_INST(JPEG, i);
110
111 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
112 ring = &adev->jpeg.inst[i].ring_dec[j];
113 ring->use_doorbell = true;
114 ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id);
115 if (!amdgpu_sriov_vf(adev)) {
116 ring->doorbell_index =
117 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
118 1 + j + 9 * jpeg_inst;
119 } else {
120 if (j < 4)
121 ring->doorbell_index =
122 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
123 4 + j + 32 * jpeg_inst;
124 else
125 ring->doorbell_index =
126 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
127 8 + j + 32 * jpeg_inst;
128 }
129 sprintf(ring->name, "jpeg_dec_%d.%d", adev->jpeg.inst[i].aid_id, j);
130 r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
131 AMDGPU_RING_PRIO_DEFAULT, NULL);
132 if (r)
133 return r;
134
135 adev->jpeg.internal.jpeg_pitch[j] =
136 regUVD_JRBC0_UVD_JRBC_SCRATCH0_INTERNAL_OFFSET;
137 adev->jpeg.inst[i].external.jpeg_pitch[j] =
138 SOC15_REG_OFFSET1(
139 JPEG, jpeg_inst,
140 regUVD_JRBC0_UVD_JRBC_SCRATCH0,
141 (j ? (0x40 * j - 0xc80) : 0));
142 }
143 }
144
145 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
146 r = amdgpu_jpeg_ras_sw_init(adev);
147 if (r) {
148 dev_err(adev->dev, "Failed to initialize jpeg ras block!\n");
149 return r;
150 }
151 }
152
153 return 0;
154 }
155
156 /**
157 * jpeg_v4_0_3_sw_fini - sw fini for JPEG block
158 *
159 * @handle: amdgpu_device pointer
160 *
161 * JPEG suspend and free up sw allocation
162 */
jpeg_v4_0_3_sw_fini(void * handle)163 static int jpeg_v4_0_3_sw_fini(void *handle)
164 {
165 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
166 int r;
167
168 r = amdgpu_jpeg_suspend(adev);
169 if (r)
170 return r;
171
172 r = amdgpu_jpeg_sw_fini(adev);
173
174 return r;
175 }
176
jpeg_v4_0_3_start_sriov(struct amdgpu_device * adev)177 static int jpeg_v4_0_3_start_sriov(struct amdgpu_device *adev)
178 {
179 struct amdgpu_ring *ring;
180 uint64_t ctx_addr;
181 uint32_t param, resp, expected;
182 uint32_t tmp, timeout;
183
184 struct amdgpu_mm_table *table = &adev->virt.mm_table;
185 uint32_t *table_loc;
186 uint32_t table_size;
187 uint32_t size, size_dw, item_offset;
188 uint32_t init_status;
189 int i, j, jpeg_inst;
190
191 struct mmsch_v4_0_cmd_direct_write
192 direct_wt = { {0} };
193 struct mmsch_v4_0_cmd_end end = { {0} };
194 struct mmsch_v4_0_3_init_header header;
195
196 direct_wt.cmd_header.command_type =
197 MMSCH_COMMAND__DIRECT_REG_WRITE;
198 end.cmd_header.command_type =
199 MMSCH_COMMAND__END;
200
201 for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {
202 jpeg_inst = GET_INST(JPEG, i);
203
204 memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header));
205 header.version = MMSCH_VERSION;
206 header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2;
207
208 table_loc = (uint32_t *)table->cpu_addr;
209 table_loc += header.total_size;
210
211 item_offset = header.total_size;
212
213 for (j = 0; j < adev->jpeg.num_jpeg_rings; j++) {
214 ring = &adev->jpeg.inst[i].ring_dec[j];
215 table_size = 0;
216
217 tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_LOW);
218 MMSCH_V4_0_INSERT_DIRECT_WT(tmp, lower_32_bits(ring->gpu_addr));
219 tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_HIGH);
220 MMSCH_V4_0_INSERT_DIRECT_WT(tmp, upper_32_bits(ring->gpu_addr));
221 tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_SIZE);
222 MMSCH_V4_0_INSERT_DIRECT_WT(tmp, ring->ring_size / 4);
223
224 if (j <= 3) {
225 header.mjpegdec0[j].table_offset = item_offset;
226 header.mjpegdec0[j].init_status = 0;
227 header.mjpegdec0[j].table_size = table_size;
228 } else {
229 header.mjpegdec1[j - 4].table_offset = item_offset;
230 header.mjpegdec1[j - 4].init_status = 0;
231 header.mjpegdec1[j - 4].table_size = table_size;
232 }
233 header.total_size += table_size;
234 item_offset += table_size;
235 }
236
237 MMSCH_V4_0_INSERT_END();
238
239 /* send init table to MMSCH */
240 size = sizeof(struct mmsch_v4_0_3_init_header);
241 table_loc = (uint32_t *)table->cpu_addr;
242 memcpy((void *)table_loc, &header, size);
243
244 ctx_addr = table->gpu_addr;
245 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
246 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
247
248 tmp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID);
249 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
250 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
251 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID, tmp);
252
253 size = header.total_size;
254 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_SIZE, size);
255
256 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP, 0);
257
258 param = 0x00000001;
259 WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_HOST, param);
260 tmp = 0;
261 timeout = 1000;
262 resp = 0;
263 expected = MMSCH_VF_MAILBOX_RESP__OK;
264 init_status =
265 ((struct mmsch_v4_0_3_init_header *)(table_loc))->mjpegdec0[i].init_status;
266 while (resp != expected) {
267 resp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP);
268
269 if (resp != 0)
270 break;
271 udelay(10);
272 tmp = tmp + 10;
273 if (tmp >= timeout) {
274 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
275 " waiting for regMMSCH_VF_MAILBOX_RESP "\
276 "(expected=0x%08x, readback=0x%08x)\n",
277 tmp, expected, resp);
278 return -EBUSY;
279 }
280 }
281 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE &&
282 init_status != MMSCH_VF_ENGINE_STATUS__PASS)
283 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init status for jpeg: %x\n",
284 resp, init_status);
285
286 }
287 return 0;
288 }
289
290 /**
291 * jpeg_v4_0_3_hw_init - start and test JPEG block
292 *
293 * @handle: amdgpu_device pointer
294 *
295 */
jpeg_v4_0_3_hw_init(void * handle)296 static int jpeg_v4_0_3_hw_init(void *handle)
297 {
298 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
299 struct amdgpu_ring *ring;
300 int i, j, r, jpeg_inst;
301
302 if (amdgpu_sriov_vf(adev)) {
303 r = jpeg_v4_0_3_start_sriov(adev);
304 if (r)
305 return r;
306
307 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
308 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
309 ring = &adev->jpeg.inst[i].ring_dec[j];
310 ring->wptr = 0;
311 ring->wptr_old = 0;
312 jpeg_v4_0_3_dec_ring_set_wptr(ring);
313 ring->sched.ready = true;
314 }
315 }
316 } else {
317 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
318 jpeg_inst = GET_INST(JPEG, i);
319
320 ring = adev->jpeg.inst[i].ring_dec;
321
322 if (ring->use_doorbell)
323 adev->nbio.funcs->vcn_doorbell_range(
324 adev, ring->use_doorbell,
325 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
326 9 * jpeg_inst,
327 adev->jpeg.inst[i].aid_id);
328
329 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
330 ring = &adev->jpeg.inst[i].ring_dec[j];
331 if (ring->use_doorbell)
332 WREG32_SOC15_OFFSET(
333 VCN, GET_INST(VCN, i),
334 regVCN_JPEG_DB_CTRL,
335 (ring->pipe ? (ring->pipe - 0x15) : 0),
336 ring->doorbell_index
337 << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
338 VCN_JPEG_DB_CTRL__EN_MASK);
339 r = amdgpu_ring_test_helper(ring);
340 if (r)
341 return r;
342 }
343 }
344 }
345 DRM_DEV_INFO(adev->dev, "JPEG decode initialized successfully.\n");
346
347 return 0;
348 }
349
350 /**
351 * jpeg_v4_0_3_hw_fini - stop the hardware block
352 *
353 * @handle: amdgpu_device pointer
354 *
355 * Stop the JPEG block, mark ring as not ready any more
356 */
jpeg_v4_0_3_hw_fini(void * handle)357 static int jpeg_v4_0_3_hw_fini(void *handle)
358 {
359 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
360 int ret = 0;
361
362 cancel_delayed_work_sync(&adev->jpeg.idle_work);
363
364 if (!amdgpu_sriov_vf(adev)) {
365 if (adev->jpeg.cur_state != AMD_PG_STATE_GATE)
366 ret = jpeg_v4_0_3_set_powergating_state(adev, AMD_PG_STATE_GATE);
367 }
368
369 return ret;
370 }
371
372 /**
373 * jpeg_v4_0_3_suspend - suspend JPEG block
374 *
375 * @handle: amdgpu_device pointer
376 *
377 * HW fini and suspend JPEG block
378 */
jpeg_v4_0_3_suspend(void * handle)379 static int jpeg_v4_0_3_suspend(void *handle)
380 {
381 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
382 int r;
383
384 r = jpeg_v4_0_3_hw_fini(adev);
385 if (r)
386 return r;
387
388 r = amdgpu_jpeg_suspend(adev);
389
390 return r;
391 }
392
393 /**
394 * jpeg_v4_0_3_resume - resume JPEG block
395 *
396 * @handle: amdgpu_device pointer
397 *
398 * Resume firmware and hw init JPEG block
399 */
jpeg_v4_0_3_resume(void * handle)400 static int jpeg_v4_0_3_resume(void *handle)
401 {
402 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
403 int r;
404
405 r = amdgpu_jpeg_resume(adev);
406 if (r)
407 return r;
408
409 r = jpeg_v4_0_3_hw_init(adev);
410
411 return r;
412 }
413
jpeg_v4_0_3_disable_clock_gating(struct amdgpu_device * adev,int inst_idx)414 static void jpeg_v4_0_3_disable_clock_gating(struct amdgpu_device *adev, int inst_idx)
415 {
416 int i, jpeg_inst;
417 uint32_t data;
418
419 jpeg_inst = GET_INST(JPEG, inst_idx);
420 data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL);
421 if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
422 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
423 data &= (~(JPEG_CGC_CTRL__JPEG0_DEC_MODE_MASK << 1));
424 } else {
425 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
426 }
427
428 data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
429 data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
430 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL, data);
431
432 data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE);
433 data &= ~(JPEG_CGC_GATE__JMCIF_MASK | JPEG_CGC_GATE__JRBBM_MASK);
434 for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i)
435 data &= ~(JPEG_CGC_GATE__JPEG0_DEC_MASK << i);
436 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE, data);
437 }
438
jpeg_v4_0_3_enable_clock_gating(struct amdgpu_device * adev,int inst_idx)439 static void jpeg_v4_0_3_enable_clock_gating(struct amdgpu_device *adev, int inst_idx)
440 {
441 int i, jpeg_inst;
442 uint32_t data;
443
444 jpeg_inst = GET_INST(JPEG, inst_idx);
445 data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL);
446 if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
447 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
448 data |= (JPEG_CGC_CTRL__JPEG0_DEC_MODE_MASK << 1);
449 } else {
450 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
451 }
452
453 data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
454 data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
455 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL, data);
456
457 data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE);
458 data |= (JPEG_CGC_GATE__JMCIF_MASK | JPEG_CGC_GATE__JRBBM_MASK);
459 for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i)
460 data |= (JPEG_CGC_GATE__JPEG0_DEC_MASK << i);
461 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE, data);
462 }
463
464 /**
465 * jpeg_v4_0_3_start - start JPEG block
466 *
467 * @adev: amdgpu_device pointer
468 *
469 * Setup and start the JPEG block
470 */
jpeg_v4_0_3_start(struct amdgpu_device * adev)471 static int jpeg_v4_0_3_start(struct amdgpu_device *adev)
472 {
473 struct amdgpu_ring *ring;
474 int i, j, jpeg_inst;
475
476 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
477 jpeg_inst = GET_INST(JPEG, i);
478
479 WREG32_SOC15(JPEG, jpeg_inst, regUVD_PGFSM_CONFIG,
480 1 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT);
481 SOC15_WAIT_ON_RREG(
482 JPEG, jpeg_inst, regUVD_PGFSM_STATUS,
483 UVD_PGFSM_STATUS__UVDJ_PWR_ON
484 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT,
485 UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
486
487 /* disable anti hang mechanism */
488 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
489 regUVD_JPEG_POWER_STATUS),
490 0, ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
491
492 /* JPEG disable CGC */
493 jpeg_v4_0_3_disable_clock_gating(adev, i);
494
495 /* MJPEG global tiling registers */
496 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_DEC_GFX8_ADDR_CONFIG,
497 adev->gfx.config.gb_addr_config);
498 WREG32_SOC15(JPEG, jpeg_inst, regJPEG_DEC_GFX10_ADDR_CONFIG,
499 adev->gfx.config.gb_addr_config);
500
501 /* enable JMI channel */
502 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL), 0,
503 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
504
505 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
506 unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
507
508 ring = &adev->jpeg.inst[i].ring_dec[j];
509
510 /* enable System Interrupt for JRBC */
511 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
512 regJPEG_SYS_INT_EN),
513 JPEG_SYS_INT_EN__DJRBC0_MASK << j,
514 ~(JPEG_SYS_INT_EN__DJRBC0_MASK << j));
515
516 WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
517 regUVD_JMI0_UVD_LMI_JRBC_RB_VMID,
518 reg_offset, 0);
519 WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
520 regUVD_JRBC0_UVD_JRBC_RB_CNTL,
521 reg_offset,
522 (0x00000001L | 0x00000002L));
523 WREG32_SOC15_OFFSET(
524 JPEG, jpeg_inst,
525 regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_LOW,
526 reg_offset, lower_32_bits(ring->gpu_addr));
527 WREG32_SOC15_OFFSET(
528 JPEG, jpeg_inst,
529 regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
530 reg_offset, upper_32_bits(ring->gpu_addr));
531 WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
532 regUVD_JRBC0_UVD_JRBC_RB_RPTR,
533 reg_offset, 0);
534 WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
535 regUVD_JRBC0_UVD_JRBC_RB_WPTR,
536 reg_offset, 0);
537 WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
538 regUVD_JRBC0_UVD_JRBC_RB_CNTL,
539 reg_offset, 0x00000002L);
540 WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
541 regUVD_JRBC0_UVD_JRBC_RB_SIZE,
542 reg_offset, ring->ring_size / 4);
543 ring->wptr = RREG32_SOC15_OFFSET(
544 JPEG, jpeg_inst, regUVD_JRBC0_UVD_JRBC_RB_WPTR,
545 reg_offset);
546 }
547 }
548
549 return 0;
550 }
551
552 /**
553 * jpeg_v4_0_3_stop - stop JPEG block
554 *
555 * @adev: amdgpu_device pointer
556 *
557 * stop the JPEG block
558 */
jpeg_v4_0_3_stop(struct amdgpu_device * adev)559 static int jpeg_v4_0_3_stop(struct amdgpu_device *adev)
560 {
561 int i, jpeg_inst;
562
563 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
564 jpeg_inst = GET_INST(JPEG, i);
565 /* reset JMI */
566 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL),
567 UVD_JMI_CNTL__SOFT_RESET_MASK,
568 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
569
570 jpeg_v4_0_3_enable_clock_gating(adev, i);
571
572 /* enable anti hang mechanism */
573 WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst,
574 regUVD_JPEG_POWER_STATUS),
575 UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
576 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
577
578 WREG32_SOC15(JPEG, jpeg_inst, regUVD_PGFSM_CONFIG,
579 2 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT);
580 SOC15_WAIT_ON_RREG(
581 JPEG, jpeg_inst, regUVD_PGFSM_STATUS,
582 UVD_PGFSM_STATUS__UVDJ_PWR_OFF
583 << UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT,
584 UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
585 }
586
587 return 0;
588 }
589
590 /**
591 * jpeg_v4_0_3_dec_ring_get_rptr - get read pointer
592 *
593 * @ring: amdgpu_ring pointer
594 *
595 * Returns the current hardware read pointer
596 */
jpeg_v4_0_3_dec_ring_get_rptr(struct amdgpu_ring * ring)597 static uint64_t jpeg_v4_0_3_dec_ring_get_rptr(struct amdgpu_ring *ring)
598 {
599 struct amdgpu_device *adev = ring->adev;
600
601 return RREG32_SOC15_OFFSET(
602 JPEG, GET_INST(JPEG, ring->me), regUVD_JRBC0_UVD_JRBC_RB_RPTR,
603 ring->pipe ? (0x40 * ring->pipe - 0xc80) : 0);
604 }
605
606 /**
607 * jpeg_v4_0_3_dec_ring_get_wptr - get write pointer
608 *
609 * @ring: amdgpu_ring pointer
610 *
611 * Returns the current hardware write pointer
612 */
jpeg_v4_0_3_dec_ring_get_wptr(struct amdgpu_ring * ring)613 static uint64_t jpeg_v4_0_3_dec_ring_get_wptr(struct amdgpu_ring *ring)
614 {
615 struct amdgpu_device *adev = ring->adev;
616
617 if (ring->use_doorbell)
618 return adev->wb.wb[ring->wptr_offs];
619 else
620 return RREG32_SOC15_OFFSET(
621 JPEG, GET_INST(JPEG, ring->me),
622 regUVD_JRBC0_UVD_JRBC_RB_WPTR,
623 ring->pipe ? (0x40 * ring->pipe - 0xc80) : 0);
624 }
625
626 /**
627 * jpeg_v4_0_3_dec_ring_set_wptr - set write pointer
628 *
629 * @ring: amdgpu_ring pointer
630 *
631 * Commits the write pointer to the hardware
632 */
jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring * ring)633 static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring)
634 {
635 struct amdgpu_device *adev = ring->adev;
636
637 if (ring->use_doorbell) {
638 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
639 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
640 } else {
641 WREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me),
642 regUVD_JRBC0_UVD_JRBC_RB_WPTR,
643 (ring->pipe ? (0x40 * ring->pipe - 0xc80) :
644 0),
645 lower_32_bits(ring->wptr));
646 }
647 }
648
649 /**
650 * jpeg_v4_0_3_dec_ring_insert_start - insert a start command
651 *
652 * @ring: amdgpu_ring pointer
653 *
654 * Write a start command to the ring.
655 */
jpeg_v4_0_3_dec_ring_insert_start(struct amdgpu_ring * ring)656 static void jpeg_v4_0_3_dec_ring_insert_start(struct amdgpu_ring *ring)
657 {
658 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
659 0, 0, PACKETJ_TYPE0));
660 amdgpu_ring_write(ring, 0x62a04); /* PCTL0_MMHUB_DEEPSLEEP_IB */
661
662 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
663 0, 0, PACKETJ_TYPE0));
664 amdgpu_ring_write(ring, 0x80004000);
665 }
666
667 /**
668 * jpeg_v4_0_3_dec_ring_insert_end - insert a end command
669 *
670 * @ring: amdgpu_ring pointer
671 *
672 * Write a end command to the ring.
673 */
jpeg_v4_0_3_dec_ring_insert_end(struct amdgpu_ring * ring)674 static void jpeg_v4_0_3_dec_ring_insert_end(struct amdgpu_ring *ring)
675 {
676 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
677 0, 0, PACKETJ_TYPE0));
678 amdgpu_ring_write(ring, 0x62a04);
679
680 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
681 0, 0, PACKETJ_TYPE0));
682 amdgpu_ring_write(ring, 0x00004000);
683 }
684
685 /**
686 * jpeg_v4_0_3_dec_ring_emit_fence - emit an fence & trap command
687 *
688 * @ring: amdgpu_ring pointer
689 * @addr: address
690 * @seq: sequence number
691 * @flags: fence related flags
692 *
693 * Write a fence and a trap command to the ring.
694 */
jpeg_v4_0_3_dec_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)695 static void jpeg_v4_0_3_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
696 unsigned int flags)
697 {
698 WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
699
700 amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET,
701 0, 0, PACKETJ_TYPE0));
702 amdgpu_ring_write(ring, seq);
703
704 amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET,
705 0, 0, PACKETJ_TYPE0));
706 amdgpu_ring_write(ring, seq);
707
708 amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET,
709 0, 0, PACKETJ_TYPE0));
710 amdgpu_ring_write(ring, lower_32_bits(addr));
711
712 amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET,
713 0, 0, PACKETJ_TYPE0));
714 amdgpu_ring_write(ring, upper_32_bits(addr));
715
716 amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
717 0, 0, PACKETJ_TYPE0));
718 amdgpu_ring_write(ring, 0x8);
719
720 amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
721 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
722 amdgpu_ring_write(ring, 0);
723
724 if (ring->adev->jpeg.inst[ring->me].aid_id) {
725 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_MCM_ADDR_INTERNAL_OFFSET,
726 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE0));
727 amdgpu_ring_write(ring, 0x4);
728 } else {
729 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
730 amdgpu_ring_write(ring, 0);
731 }
732
733 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
734 0, 0, PACKETJ_TYPE0));
735 amdgpu_ring_write(ring, 0x3fbc);
736
737 if (ring->adev->jpeg.inst[ring->me].aid_id) {
738 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_MCM_ADDR_INTERNAL_OFFSET,
739 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE0));
740 amdgpu_ring_write(ring, 0x0);
741 } else {
742 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
743 amdgpu_ring_write(ring, 0);
744 }
745
746 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
747 0, 0, PACKETJ_TYPE0));
748 amdgpu_ring_write(ring, 0x1);
749
750 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
751 amdgpu_ring_write(ring, 0);
752 }
753
754 /**
755 * jpeg_v4_0_3_dec_ring_emit_ib - execute indirect buffer
756 *
757 * @ring: amdgpu_ring pointer
758 * @job: job to retrieve vmid from
759 * @ib: indirect buffer to execute
760 * @flags: unused
761 *
762 * Write ring commands to execute the indirect buffer.
763 */
jpeg_v4_0_3_dec_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)764 static void jpeg_v4_0_3_dec_ring_emit_ib(struct amdgpu_ring *ring,
765 struct amdgpu_job *job,
766 struct amdgpu_ib *ib,
767 uint32_t flags)
768 {
769 unsigned int vmid = AMDGPU_JOB_GET_VMID(job);
770
771 amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET,
772 0, 0, PACKETJ_TYPE0));
773
774 if (ring->funcs->parse_cs)
775 amdgpu_ring_write(ring, 0);
776 else
777 amdgpu_ring_write(ring, (vmid | (vmid << 4) | (vmid << 8)));
778
779 amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JPEG_VMID_INTERNAL_OFFSET,
780 0, 0, PACKETJ_TYPE0));
781 amdgpu_ring_write(ring, (vmid | (vmid << 4) | (vmid << 8)));
782
783 amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET,
784 0, 0, PACKETJ_TYPE0));
785 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
786
787 amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET,
788 0, 0, PACKETJ_TYPE0));
789 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
790
791 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_IB_SIZE_INTERNAL_OFFSET,
792 0, 0, PACKETJ_TYPE0));
793 amdgpu_ring_write(ring, ib->length_dw);
794
795 amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET,
796 0, 0, PACKETJ_TYPE0));
797 amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
798
799 amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET,
800 0, 0, PACKETJ_TYPE0));
801 amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
802
803 amdgpu_ring_write(ring, PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
804 amdgpu_ring_write(ring, 0);
805
806 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
807 0, 0, PACKETJ_TYPE0));
808 amdgpu_ring_write(ring, 0x01400200);
809
810 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
811 0, 0, PACKETJ_TYPE0));
812 amdgpu_ring_write(ring, 0x2);
813
814 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_STATUS_INTERNAL_OFFSET,
815 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
816 amdgpu_ring_write(ring, 0x2);
817 }
818
jpeg_v4_0_3_dec_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)819 static void jpeg_v4_0_3_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
820 uint32_t val, uint32_t mask)
821 {
822 uint32_t reg_offset = (reg << 2);
823
824 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
825 0, 0, PACKETJ_TYPE0));
826 amdgpu_ring_write(ring, 0x01400200);
827
828 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
829 0, 0, PACKETJ_TYPE0));
830 amdgpu_ring_write(ring, val);
831
832 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
833 0, 0, PACKETJ_TYPE0));
834 if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
835 amdgpu_ring_write(ring, 0);
836 amdgpu_ring_write(ring,
837 PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
838 } else {
839 amdgpu_ring_write(ring, reg_offset);
840 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
841 0, 0, PACKETJ_TYPE3));
842 }
843 amdgpu_ring_write(ring, mask);
844 }
845
jpeg_v4_0_3_dec_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned int vmid,uint64_t pd_addr)846 static void jpeg_v4_0_3_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
847 unsigned int vmid, uint64_t pd_addr)
848 {
849 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
850 uint32_t data0, data1, mask;
851
852 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
853
854 /* wait for register write */
855 data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
856 data1 = lower_32_bits(pd_addr);
857 mask = 0xffffffff;
858 jpeg_v4_0_3_dec_ring_emit_reg_wait(ring, data0, data1, mask);
859 }
860
jpeg_v4_0_3_dec_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)861 static void jpeg_v4_0_3_dec_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
862 {
863 uint32_t reg_offset = (reg << 2);
864
865 amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
866 0, 0, PACKETJ_TYPE0));
867 if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
868 amdgpu_ring_write(ring, 0);
869 amdgpu_ring_write(ring,
870 PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
871 } else {
872 amdgpu_ring_write(ring, reg_offset);
873 amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
874 0, 0, PACKETJ_TYPE0));
875 }
876 amdgpu_ring_write(ring, val);
877 }
878
jpeg_v4_0_3_dec_ring_nop(struct amdgpu_ring * ring,uint32_t count)879 static void jpeg_v4_0_3_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count)
880 {
881 int i;
882
883 WARN_ON(ring->wptr % 2 || count % 2);
884
885 for (i = 0; i < count / 2; i++) {
886 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
887 amdgpu_ring_write(ring, 0);
888 }
889 }
890
jpeg_v4_0_3_is_idle(void * handle)891 static bool jpeg_v4_0_3_is_idle(void *handle)
892 {
893 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
894 bool ret = false;
895 int i, j;
896
897 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
898 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
899 unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
900
901 ret &= ((RREG32_SOC15_OFFSET(
902 JPEG, GET_INST(JPEG, i),
903 regUVD_JRBC0_UVD_JRBC_STATUS,
904 reg_offset) &
905 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
906 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
907 }
908 }
909
910 return ret;
911 }
912
jpeg_v4_0_3_wait_for_idle(void * handle)913 static int jpeg_v4_0_3_wait_for_idle(void *handle)
914 {
915 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
916 int ret = 0;
917 int i, j;
918
919 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
920 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
921 unsigned int reg_offset = (j?(0x40 * j - 0xc80):0);
922
923 ret &= SOC15_WAIT_ON_RREG_OFFSET(
924 JPEG, GET_INST(JPEG, i),
925 regUVD_JRBC0_UVD_JRBC_STATUS, reg_offset,
926 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
927 UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
928 }
929 }
930 return ret;
931 }
932
jpeg_v4_0_3_set_clockgating_state(void * handle,enum amd_clockgating_state state)933 static int jpeg_v4_0_3_set_clockgating_state(void *handle,
934 enum amd_clockgating_state state)
935 {
936 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
937 bool enable = state == AMD_CG_STATE_GATE;
938 int i;
939
940 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
941 if (enable) {
942 if (!jpeg_v4_0_3_is_idle(handle))
943 return -EBUSY;
944 jpeg_v4_0_3_enable_clock_gating(adev, i);
945 } else {
946 jpeg_v4_0_3_disable_clock_gating(adev, i);
947 }
948 }
949 return 0;
950 }
951
jpeg_v4_0_3_set_powergating_state(void * handle,enum amd_powergating_state state)952 static int jpeg_v4_0_3_set_powergating_state(void *handle,
953 enum amd_powergating_state state)
954 {
955 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
956 int ret;
957
958 if (state == adev->jpeg.cur_state)
959 return 0;
960
961 if (state == AMD_PG_STATE_GATE)
962 ret = jpeg_v4_0_3_stop(adev);
963 else
964 ret = jpeg_v4_0_3_start(adev);
965
966 if (!ret)
967 adev->jpeg.cur_state = state;
968
969 return ret;
970 }
971
jpeg_v4_0_3_set_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)972 static int jpeg_v4_0_3_set_interrupt_state(struct amdgpu_device *adev,
973 struct amdgpu_irq_src *source,
974 unsigned int type,
975 enum amdgpu_interrupt_state state)
976 {
977 return 0;
978 }
979
jpeg_v4_0_3_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)980 static int jpeg_v4_0_3_process_interrupt(struct amdgpu_device *adev,
981 struct amdgpu_irq_src *source,
982 struct amdgpu_iv_entry *entry)
983 {
984 uint32_t i, inst;
985
986 i = node_id_to_phys_map[entry->node_id];
987 DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n");
988
989 for (inst = 0; inst < adev->jpeg.num_jpeg_inst; ++inst)
990 if (adev->jpeg.inst[inst].aid_id == i)
991 break;
992
993 if (inst >= adev->jpeg.num_jpeg_inst) {
994 dev_WARN_ONCE(adev->dev, 1,
995 "Interrupt received for unknown JPEG instance %d",
996 entry->node_id);
997 return 0;
998 }
999
1000 switch (entry->src_id) {
1001 case VCN_4_0__SRCID__JPEG_DECODE:
1002 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[0]);
1003 break;
1004 case VCN_4_0__SRCID__JPEG1_DECODE:
1005 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[1]);
1006 break;
1007 case VCN_4_0__SRCID__JPEG2_DECODE:
1008 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[2]);
1009 break;
1010 case VCN_4_0__SRCID__JPEG3_DECODE:
1011 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
1012 break;
1013 case VCN_4_0__SRCID__JPEG4_DECODE:
1014 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
1015 break;
1016 case VCN_4_0__SRCID__JPEG5_DECODE:
1017 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
1018 break;
1019 case VCN_4_0__SRCID__JPEG6_DECODE:
1020 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
1021 break;
1022 case VCN_4_0__SRCID__JPEG7_DECODE:
1023 amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
1024 break;
1025 default:
1026 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
1027 entry->src_id, entry->src_data[0]);
1028 break;
1029 }
1030
1031 return 0;
1032 }
1033
1034 static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = {
1035 .name = "jpeg_v4_0_3",
1036 .early_init = jpeg_v4_0_3_early_init,
1037 .late_init = NULL,
1038 .sw_init = jpeg_v4_0_3_sw_init,
1039 .sw_fini = jpeg_v4_0_3_sw_fini,
1040 .hw_init = jpeg_v4_0_3_hw_init,
1041 .hw_fini = jpeg_v4_0_3_hw_fini,
1042 .suspend = jpeg_v4_0_3_suspend,
1043 .resume = jpeg_v4_0_3_resume,
1044 .is_idle = jpeg_v4_0_3_is_idle,
1045 .wait_for_idle = jpeg_v4_0_3_wait_for_idle,
1046 .check_soft_reset = NULL,
1047 .pre_soft_reset = NULL,
1048 .soft_reset = NULL,
1049 .post_soft_reset = NULL,
1050 .set_clockgating_state = jpeg_v4_0_3_set_clockgating_state,
1051 .set_powergating_state = jpeg_v4_0_3_set_powergating_state,
1052 };
1053
1054 static const struct amdgpu_ring_funcs jpeg_v4_0_3_dec_ring_vm_funcs = {
1055 .type = AMDGPU_RING_TYPE_VCN_JPEG,
1056 .align_mask = 0xf,
1057 .get_rptr = jpeg_v4_0_3_dec_ring_get_rptr,
1058 .get_wptr = jpeg_v4_0_3_dec_ring_get_wptr,
1059 .set_wptr = jpeg_v4_0_3_dec_ring_set_wptr,
1060 .parse_cs = jpeg_v4_0_3_dec_ring_parse_cs,
1061 .emit_frame_size =
1062 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1063 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1064 8 + /* jpeg_v4_0_3_dec_ring_emit_vm_flush */
1065 22 + 22 + /* jpeg_v4_0_3_dec_ring_emit_fence x2 vm fence */
1066 8 + 16,
1067 .emit_ib_size = 22, /* jpeg_v4_0_3_dec_ring_emit_ib */
1068 .emit_ib = jpeg_v4_0_3_dec_ring_emit_ib,
1069 .emit_fence = jpeg_v4_0_3_dec_ring_emit_fence,
1070 .emit_vm_flush = jpeg_v4_0_3_dec_ring_emit_vm_flush,
1071 .test_ring = amdgpu_jpeg_dec_ring_test_ring,
1072 .test_ib = amdgpu_jpeg_dec_ring_test_ib,
1073 .insert_nop = jpeg_v4_0_3_dec_ring_nop,
1074 .insert_start = jpeg_v4_0_3_dec_ring_insert_start,
1075 .insert_end = jpeg_v4_0_3_dec_ring_insert_end,
1076 .pad_ib = amdgpu_ring_generic_pad_ib,
1077 .begin_use = amdgpu_jpeg_ring_begin_use,
1078 .end_use = amdgpu_jpeg_ring_end_use,
1079 .emit_wreg = jpeg_v4_0_3_dec_ring_emit_wreg,
1080 .emit_reg_wait = jpeg_v4_0_3_dec_ring_emit_reg_wait,
1081 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1082 };
1083
jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device * adev)1084 static void jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device *adev)
1085 {
1086 int i, j, jpeg_inst;
1087
1088 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1089 for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
1090 adev->jpeg.inst[i].ring_dec[j].funcs = &jpeg_v4_0_3_dec_ring_vm_funcs;
1091 adev->jpeg.inst[i].ring_dec[j].me = i;
1092 adev->jpeg.inst[i].ring_dec[j].pipe = j;
1093 }
1094 jpeg_inst = GET_INST(JPEG, i);
1095 adev->jpeg.inst[i].aid_id =
1096 jpeg_inst / adev->jpeg.num_inst_per_aid;
1097 }
1098 DRM_DEV_INFO(adev->dev, "JPEG decode is enabled in VM mode\n");
1099 }
1100
1101 static const struct amdgpu_irq_src_funcs jpeg_v4_0_3_irq_funcs = {
1102 .set = jpeg_v4_0_3_set_interrupt_state,
1103 .process = jpeg_v4_0_3_process_interrupt,
1104 };
1105
jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device * adev)1106 static void jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device *adev)
1107 {
1108 int i;
1109
1110 for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1111 adev->jpeg.inst->irq.num_types += adev->jpeg.num_jpeg_rings;
1112 }
1113 adev->jpeg.inst->irq.funcs = &jpeg_v4_0_3_irq_funcs;
1114 }
1115
1116 const struct amdgpu_ip_block_version jpeg_v4_0_3_ip_block = {
1117 .type = AMD_IP_BLOCK_TYPE_JPEG,
1118 .major = 4,
1119 .minor = 0,
1120 .rev = 3,
1121 .funcs = &jpeg_v4_0_3_ip_funcs,
1122 };
1123
1124 static const struct amdgpu_ras_err_status_reg_entry jpeg_v4_0_3_ue_reg_list[] = {
1125 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG0S, regVCN_UE_ERR_STATUS_HI_JPEG0S),
1126 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG0S"},
1127 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG0D, regVCN_UE_ERR_STATUS_HI_JPEG0D),
1128 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG0D"},
1129 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG1S, regVCN_UE_ERR_STATUS_HI_JPEG1S),
1130 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG1S"},
1131 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG1D, regVCN_UE_ERR_STATUS_HI_JPEG1D),
1132 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG1D"},
1133 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG2S, regVCN_UE_ERR_STATUS_HI_JPEG2S),
1134 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG2S"},
1135 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG2D, regVCN_UE_ERR_STATUS_HI_JPEG2D),
1136 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG2D"},
1137 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG3S, regVCN_UE_ERR_STATUS_HI_JPEG3S),
1138 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG3S"},
1139 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG3D, regVCN_UE_ERR_STATUS_HI_JPEG3D),
1140 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG3D"},
1141 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG4S, regVCN_UE_ERR_STATUS_HI_JPEG4S),
1142 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG4S"},
1143 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG4D, regVCN_UE_ERR_STATUS_HI_JPEG4D),
1144 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG4D"},
1145 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG5S, regVCN_UE_ERR_STATUS_HI_JPEG5S),
1146 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG5S"},
1147 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG5D, regVCN_UE_ERR_STATUS_HI_JPEG5D),
1148 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG5D"},
1149 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG6S, regVCN_UE_ERR_STATUS_HI_JPEG6S),
1150 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG6S"},
1151 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG6D, regVCN_UE_ERR_STATUS_HI_JPEG6D),
1152 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG6D"},
1153 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG7S, regVCN_UE_ERR_STATUS_HI_JPEG7S),
1154 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG7S"},
1155 {AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG7D, regVCN_UE_ERR_STATUS_HI_JPEG7D),
1156 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG7D"},
1157 };
1158
jpeg_v4_0_3_inst_query_ras_error_count(struct amdgpu_device * adev,uint32_t jpeg_inst,void * ras_err_status)1159 static void jpeg_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev,
1160 uint32_t jpeg_inst,
1161 void *ras_err_status)
1162 {
1163 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
1164
1165 /* jpeg v4_0_3 only support uncorrectable errors */
1166 amdgpu_ras_inst_query_ras_error_count(adev,
1167 jpeg_v4_0_3_ue_reg_list,
1168 ARRAY_SIZE(jpeg_v4_0_3_ue_reg_list),
1169 NULL, 0, GET_INST(VCN, jpeg_inst),
1170 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
1171 &err_data->ue_count);
1172 }
1173
jpeg_v4_0_3_query_ras_error_count(struct amdgpu_device * adev,void * ras_err_status)1174 static void jpeg_v4_0_3_query_ras_error_count(struct amdgpu_device *adev,
1175 void *ras_err_status)
1176 {
1177 uint32_t i;
1178
1179 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
1180 dev_warn(adev->dev, "JPEG RAS is not supported\n");
1181 return;
1182 }
1183
1184 for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
1185 jpeg_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status);
1186 }
1187
jpeg_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device * adev,uint32_t jpeg_inst)1188 static void jpeg_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev,
1189 uint32_t jpeg_inst)
1190 {
1191 amdgpu_ras_inst_reset_ras_error_count(adev,
1192 jpeg_v4_0_3_ue_reg_list,
1193 ARRAY_SIZE(jpeg_v4_0_3_ue_reg_list),
1194 GET_INST(VCN, jpeg_inst));
1195 }
1196
jpeg_v4_0_3_reset_ras_error_count(struct amdgpu_device * adev)1197 static void jpeg_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev)
1198 {
1199 uint32_t i;
1200
1201 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
1202 dev_warn(adev->dev, "JPEG RAS is not supported\n");
1203 return;
1204 }
1205
1206 for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
1207 jpeg_v4_0_3_inst_reset_ras_error_count(adev, i);
1208 }
1209
1210 static const struct amdgpu_ras_block_hw_ops jpeg_v4_0_3_ras_hw_ops = {
1211 .query_ras_error_count = jpeg_v4_0_3_query_ras_error_count,
1212 .reset_ras_error_count = jpeg_v4_0_3_reset_ras_error_count,
1213 };
1214
1215 static struct amdgpu_jpeg_ras jpeg_v4_0_3_ras = {
1216 .ras_block = {
1217 .hw_ops = &jpeg_v4_0_3_ras_hw_ops,
1218 },
1219 };
1220
jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device * adev)1221 static void jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device *adev)
1222 {
1223 adev->jpeg.ras = &jpeg_v4_0_3_ras;
1224 }
1225
1226 /**
1227 * jpeg_v4_0_3_dec_ring_parse_cs - command submission parser
1228 *
1229 * @parser: Command submission parser context
1230 * @job: the job to parse
1231 * @ib: the IB to parse
1232 *
1233 * Parse the command stream, return -EINVAL for invalid packet,
1234 * 0 otherwise
1235 */
jpeg_v4_0_3_dec_ring_parse_cs(struct amdgpu_cs_parser * parser,struct amdgpu_job * job,struct amdgpu_ib * ib)1236 int jpeg_v4_0_3_dec_ring_parse_cs(struct amdgpu_cs_parser *parser,
1237 struct amdgpu_job *job,
1238 struct amdgpu_ib *ib)
1239 {
1240 uint32_t i, reg, res, cond, type;
1241 struct amdgpu_device *adev = parser->adev;
1242
1243 for (i = 0; i < ib->length_dw ; i += 2) {
1244 reg = CP_PACKETJ_GET_REG(ib->ptr[i]);
1245 res = CP_PACKETJ_GET_RES(ib->ptr[i]);
1246 cond = CP_PACKETJ_GET_COND(ib->ptr[i]);
1247 type = CP_PACKETJ_GET_TYPE(ib->ptr[i]);
1248
1249 if (res) /* only support 0 at the moment */
1250 return -EINVAL;
1251
1252 switch (type) {
1253 case PACKETJ_TYPE0:
1254 if (cond != PACKETJ_CONDITION_CHECK0 || reg < JPEG_REG_RANGE_START || reg > JPEG_REG_RANGE_END) {
1255 dev_err(adev->dev, "Invalid packet [0x%08x]!\n", ib->ptr[i]);
1256 return -EINVAL;
1257 }
1258 break;
1259 case PACKETJ_TYPE3:
1260 if (cond != PACKETJ_CONDITION_CHECK3 || reg < JPEG_REG_RANGE_START || reg > JPEG_REG_RANGE_END) {
1261 dev_err(adev->dev, "Invalid packet [0x%08x]!\n", ib->ptr[i]);
1262 return -EINVAL;
1263 }
1264 break;
1265 case PACKETJ_TYPE6:
1266 if (ib->ptr[i] == CP_PACKETJ_NOP)
1267 continue;
1268 dev_err(adev->dev, "Invalid packet [0x%08x]!\n", ib->ptr[i]);
1269 return -EINVAL;
1270 default:
1271 dev_err(adev->dev, "Unknown packet type %d !\n", type);
1272 return -EINVAL;
1273 }
1274 }
1275
1276 return 0;
1277 }
1278