xref: /openbmc/linux/drivers/gpu/drm/i915/gvt/cmd_parser.c (revision f645d22b)
1 /*
2  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21  * SOFTWARE.
22  *
23  * Authors:
24  *    Ke Yu
25  *    Kevin Tian <kevin.tian@intel.com>
26  *    Zhiyuan Lv <zhiyuan.lv@intel.com>
27  *
28  * Contributors:
29  *    Min He <min.he@intel.com>
30  *    Ping Gao <ping.a.gao@intel.com>
31  *    Tina Zhang <tina.zhang@intel.com>
32  *    Yulei Zhang <yulei.zhang@intel.com>
33  *    Zhi Wang <zhi.a.wang@intel.com>
34  *
35  */
36 
37 #include <linux/slab.h>
38 
39 #include "i915_drv.h"
40 #include "i915_reg.h"
41 #include "gt/intel_engine_regs.h"
42 #include "gt/intel_gpu_commands.h"
43 #include "gt/intel_gt_regs.h"
44 #include "gt/intel_lrc.h"
45 #include "gt/intel_ring.h"
46 #include "gt/intel_gt_requests.h"
47 #include "gt/shmem_utils.h"
48 #include "gvt.h"
49 #include "i915_pvinfo.h"
50 #include "trace.h"
51 
52 #include "display/intel_display.h"
53 #include "gem/i915_gem_context.h"
54 #include "gem/i915_gem_pm.h"
55 #include "gt/intel_context.h"
56 
57 #define INVALID_OP    (~0U)
58 
59 #define OP_LEN_MI           9
60 #define OP_LEN_2D           10
61 #define OP_LEN_3D_MEDIA     16
62 #define OP_LEN_MFX_VC       16
63 #define OP_LEN_VEBOX	    16
64 
65 #define CMD_TYPE(cmd)	(((cmd) >> 29) & 7)
66 
67 struct sub_op_bits {
68 	int hi;
69 	int low;
70 };
71 struct decode_info {
72 	const char *name;
73 	int op_len;
74 	int nr_sub_op;
75 	const struct sub_op_bits *sub_op;
76 };
77 
78 #define   MAX_CMD_BUDGET			0x7fffffff
79 #define   MI_WAIT_FOR_PLANE_C_FLIP_PENDING      (1<<15)
80 #define   MI_WAIT_FOR_PLANE_B_FLIP_PENDING      (1<<9)
81 #define   MI_WAIT_FOR_PLANE_A_FLIP_PENDING      (1<<1)
82 
83 #define   MI_WAIT_FOR_SPRITE_C_FLIP_PENDING      (1<<20)
84 #define   MI_WAIT_FOR_SPRITE_B_FLIP_PENDING      (1<<10)
85 #define   MI_WAIT_FOR_SPRITE_A_FLIP_PENDING      (1<<2)
86 
87 /* Render Command Map */
88 
89 /* MI_* command Opcode (28:23) */
90 #define OP_MI_NOOP                          0x0
91 #define OP_MI_SET_PREDICATE                 0x1  /* HSW+ */
92 #define OP_MI_USER_INTERRUPT                0x2
93 #define OP_MI_WAIT_FOR_EVENT                0x3
94 #define OP_MI_FLUSH                         0x4
95 #define OP_MI_ARB_CHECK                     0x5
96 #define OP_MI_RS_CONTROL                    0x6  /* HSW+ */
97 #define OP_MI_REPORT_HEAD                   0x7
98 #define OP_MI_ARB_ON_OFF                    0x8
99 #define OP_MI_URB_ATOMIC_ALLOC              0x9  /* HSW+ */
100 #define OP_MI_BATCH_BUFFER_END              0xA
101 #define OP_MI_SUSPEND_FLUSH                 0xB
102 #define OP_MI_PREDICATE                     0xC  /* IVB+ */
103 #define OP_MI_TOPOLOGY_FILTER               0xD  /* IVB+ */
104 #define OP_MI_SET_APPID                     0xE  /* IVB+ */
105 #define OP_MI_RS_CONTEXT                    0xF  /* HSW+ */
106 #define OP_MI_LOAD_SCAN_LINES_INCL          0x12 /* HSW+ */
107 #define OP_MI_DISPLAY_FLIP                  0x14
108 #define OP_MI_SEMAPHORE_MBOX                0x16
109 #define OP_MI_SET_CONTEXT                   0x18
110 #define OP_MI_MATH                          0x1A
111 #define OP_MI_URB_CLEAR                     0x19
112 #define OP_MI_SEMAPHORE_SIGNAL		    0x1B  /* BDW+ */
113 #define OP_MI_SEMAPHORE_WAIT		    0x1C  /* BDW+ */
114 
115 #define OP_MI_STORE_DATA_IMM                0x20
116 #define OP_MI_STORE_DATA_INDEX              0x21
117 #define OP_MI_LOAD_REGISTER_IMM             0x22
118 #define OP_MI_UPDATE_GTT                    0x23
119 #define OP_MI_STORE_REGISTER_MEM            0x24
120 #define OP_MI_FLUSH_DW                      0x26
121 #define OP_MI_CLFLUSH                       0x27
122 #define OP_MI_REPORT_PERF_COUNT             0x28
123 #define OP_MI_LOAD_REGISTER_MEM             0x29  /* HSW+ */
124 #define OP_MI_LOAD_REGISTER_REG             0x2A  /* HSW+ */
125 #define OP_MI_RS_STORE_DATA_IMM             0x2B  /* HSW+ */
126 #define OP_MI_LOAD_URB_MEM                  0x2C  /* HSW+ */
127 #define OP_MI_STORE_URM_MEM                 0x2D  /* HSW+ */
128 #define OP_MI_2E			    0x2E  /* BDW+ */
129 #define OP_MI_2F			    0x2F  /* BDW+ */
130 #define OP_MI_BATCH_BUFFER_START            0x31
131 
132 /* Bit definition for dword 0 */
133 #define _CMDBIT_BB_START_IN_PPGTT	(1UL << 8)
134 
135 #define OP_MI_CONDITIONAL_BATCH_BUFFER_END  0x36
136 
137 #define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2))
138 #define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U))
139 #define BATCH_BUFFER_ADR_SPACE_BIT(x)	(((x) >> 8) & 1U)
140 #define BATCH_BUFFER_2ND_LEVEL_BIT(x)   ((x) >> 22 & 1U)
141 
142 /* 2D command: Opcode (28:22) */
143 #define OP_2D(x)    ((2<<7) | x)
144 
145 #define OP_XY_SETUP_BLT                             OP_2D(0x1)
146 #define OP_XY_SETUP_CLIP_BLT                        OP_2D(0x3)
147 #define OP_XY_SETUP_MONO_PATTERN_SL_BLT             OP_2D(0x11)
148 #define OP_XY_PIXEL_BLT                             OP_2D(0x24)
149 #define OP_XY_SCANLINES_BLT                         OP_2D(0x25)
150 #define OP_XY_TEXT_BLT                              OP_2D(0x26)
151 #define OP_XY_TEXT_IMMEDIATE_BLT                    OP_2D(0x31)
152 #define OP_XY_COLOR_BLT                             OP_2D(0x50)
153 #define OP_XY_PAT_BLT                               OP_2D(0x51)
154 #define OP_XY_MONO_PAT_BLT                          OP_2D(0x52)
155 #define OP_XY_SRC_COPY_BLT                          OP_2D(0x53)
156 #define OP_XY_MONO_SRC_COPY_BLT                     OP_2D(0x54)
157 #define OP_XY_FULL_BLT                              OP_2D(0x55)
158 #define OP_XY_FULL_MONO_SRC_BLT                     OP_2D(0x56)
159 #define OP_XY_FULL_MONO_PATTERN_BLT                 OP_2D(0x57)
160 #define OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT        OP_2D(0x58)
161 #define OP_XY_MONO_PAT_FIXED_BLT                    OP_2D(0x59)
162 #define OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT           OP_2D(0x71)
163 #define OP_XY_PAT_BLT_IMMEDIATE                     OP_2D(0x72)
164 #define OP_XY_SRC_COPY_CHROMA_BLT                   OP_2D(0x73)
165 #define OP_XY_FULL_IMMEDIATE_PATTERN_BLT            OP_2D(0x74)
166 #define OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT   OP_2D(0x75)
167 #define OP_XY_PAT_CHROMA_BLT                        OP_2D(0x76)
168 #define OP_XY_PAT_CHROMA_BLT_IMMEDIATE              OP_2D(0x77)
169 
170 /* 3D/Media Command: Pipeline Type(28:27) Opcode(26:24) Sub Opcode(23:16) */
171 #define OP_3D_MEDIA(sub_type, opcode, sub_opcode) \
172 	((3 << 13) | ((sub_type) << 11) | ((opcode) << 8) | (sub_opcode))
173 
174 #define OP_STATE_PREFETCH                       OP_3D_MEDIA(0x0, 0x0, 0x03)
175 
176 #define OP_STATE_BASE_ADDRESS                   OP_3D_MEDIA(0x0, 0x1, 0x01)
177 #define OP_STATE_SIP                            OP_3D_MEDIA(0x0, 0x1, 0x02)
178 #define OP_3D_MEDIA_0_1_4			OP_3D_MEDIA(0x0, 0x1, 0x04)
179 #define OP_SWTESS_BASE_ADDRESS			OP_3D_MEDIA(0x0, 0x1, 0x03)
180 
181 #define OP_3DSTATE_VF_STATISTICS_GM45           OP_3D_MEDIA(0x1, 0x0, 0x0B)
182 
183 #define OP_PIPELINE_SELECT                      OP_3D_MEDIA(0x1, 0x1, 0x04)
184 
185 #define OP_MEDIA_VFE_STATE                      OP_3D_MEDIA(0x2, 0x0, 0x0)
186 #define OP_MEDIA_CURBE_LOAD                     OP_3D_MEDIA(0x2, 0x0, 0x1)
187 #define OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD      OP_3D_MEDIA(0x2, 0x0, 0x2)
188 #define OP_MEDIA_GATEWAY_STATE                  OP_3D_MEDIA(0x2, 0x0, 0x3)
189 #define OP_MEDIA_STATE_FLUSH                    OP_3D_MEDIA(0x2, 0x0, 0x4)
190 #define OP_MEDIA_POOL_STATE                     OP_3D_MEDIA(0x2, 0x0, 0x5)
191 
192 #define OP_MEDIA_OBJECT                         OP_3D_MEDIA(0x2, 0x1, 0x0)
193 #define OP_MEDIA_OBJECT_PRT                     OP_3D_MEDIA(0x2, 0x1, 0x2)
194 #define OP_MEDIA_OBJECT_WALKER                  OP_3D_MEDIA(0x2, 0x1, 0x3)
195 #define OP_GPGPU_WALKER                         OP_3D_MEDIA(0x2, 0x1, 0x5)
196 
197 #define OP_3DSTATE_CLEAR_PARAMS                 OP_3D_MEDIA(0x3, 0x0, 0x04) /* IVB+ */
198 #define OP_3DSTATE_DEPTH_BUFFER                 OP_3D_MEDIA(0x3, 0x0, 0x05) /* IVB+ */
199 #define OP_3DSTATE_STENCIL_BUFFER               OP_3D_MEDIA(0x3, 0x0, 0x06) /* IVB+ */
200 #define OP_3DSTATE_HIER_DEPTH_BUFFER            OP_3D_MEDIA(0x3, 0x0, 0x07) /* IVB+ */
201 #define OP_3DSTATE_VERTEX_BUFFERS               OP_3D_MEDIA(0x3, 0x0, 0x08)
202 #define OP_3DSTATE_VERTEX_ELEMENTS              OP_3D_MEDIA(0x3, 0x0, 0x09)
203 #define OP_3DSTATE_INDEX_BUFFER                 OP_3D_MEDIA(0x3, 0x0, 0x0A)
204 #define OP_3DSTATE_VF_STATISTICS                OP_3D_MEDIA(0x3, 0x0, 0x0B)
205 #define OP_3DSTATE_VF                           OP_3D_MEDIA(0x3, 0x0, 0x0C)  /* HSW+ */
206 #define OP_3DSTATE_CC_STATE_POINTERS            OP_3D_MEDIA(0x3, 0x0, 0x0E)
207 #define OP_3DSTATE_SCISSOR_STATE_POINTERS       OP_3D_MEDIA(0x3, 0x0, 0x0F)
208 #define OP_3DSTATE_VS                           OP_3D_MEDIA(0x3, 0x0, 0x10)
209 #define OP_3DSTATE_GS                           OP_3D_MEDIA(0x3, 0x0, 0x11)
210 #define OP_3DSTATE_CLIP                         OP_3D_MEDIA(0x3, 0x0, 0x12)
211 #define OP_3DSTATE_SF                           OP_3D_MEDIA(0x3, 0x0, 0x13)
212 #define OP_3DSTATE_WM                           OP_3D_MEDIA(0x3, 0x0, 0x14)
213 #define OP_3DSTATE_CONSTANT_VS                  OP_3D_MEDIA(0x3, 0x0, 0x15)
214 #define OP_3DSTATE_CONSTANT_GS                  OP_3D_MEDIA(0x3, 0x0, 0x16)
215 #define OP_3DSTATE_CONSTANT_PS                  OP_3D_MEDIA(0x3, 0x0, 0x17)
216 #define OP_3DSTATE_SAMPLE_MASK                  OP_3D_MEDIA(0x3, 0x0, 0x18)
217 #define OP_3DSTATE_CONSTANT_HS                  OP_3D_MEDIA(0x3, 0x0, 0x19) /* IVB+ */
218 #define OP_3DSTATE_CONSTANT_DS                  OP_3D_MEDIA(0x3, 0x0, 0x1A) /* IVB+ */
219 #define OP_3DSTATE_HS                           OP_3D_MEDIA(0x3, 0x0, 0x1B) /* IVB+ */
220 #define OP_3DSTATE_TE                           OP_3D_MEDIA(0x3, 0x0, 0x1C) /* IVB+ */
221 #define OP_3DSTATE_DS                           OP_3D_MEDIA(0x3, 0x0, 0x1D) /* IVB+ */
222 #define OP_3DSTATE_STREAMOUT                    OP_3D_MEDIA(0x3, 0x0, 0x1E) /* IVB+ */
223 #define OP_3DSTATE_SBE                          OP_3D_MEDIA(0x3, 0x0, 0x1F) /* IVB+ */
224 #define OP_3DSTATE_PS                           OP_3D_MEDIA(0x3, 0x0, 0x20) /* IVB+ */
225 #define OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP OP_3D_MEDIA(0x3, 0x0, 0x21) /* IVB+ */
226 #define OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC   OP_3D_MEDIA(0x3, 0x0, 0x23) /* IVB+ */
227 #define OP_3DSTATE_BLEND_STATE_POINTERS         OP_3D_MEDIA(0x3, 0x0, 0x24) /* IVB+ */
228 #define OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x25) /* IVB+ */
229 #define OP_3DSTATE_BINDING_TABLE_POINTERS_VS    OP_3D_MEDIA(0x3, 0x0, 0x26) /* IVB+ */
230 #define OP_3DSTATE_BINDING_TABLE_POINTERS_HS    OP_3D_MEDIA(0x3, 0x0, 0x27) /* IVB+ */
231 #define OP_3DSTATE_BINDING_TABLE_POINTERS_DS    OP_3D_MEDIA(0x3, 0x0, 0x28) /* IVB+ */
232 #define OP_3DSTATE_BINDING_TABLE_POINTERS_GS    OP_3D_MEDIA(0x3, 0x0, 0x29) /* IVB+ */
233 #define OP_3DSTATE_BINDING_TABLE_POINTERS_PS    OP_3D_MEDIA(0x3, 0x0, 0x2A) /* IVB+ */
234 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_VS    OP_3D_MEDIA(0x3, 0x0, 0x2B) /* IVB+ */
235 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_HS    OP_3D_MEDIA(0x3, 0x0, 0x2C) /* IVB+ */
236 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_DS    OP_3D_MEDIA(0x3, 0x0, 0x2D) /* IVB+ */
237 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_GS    OP_3D_MEDIA(0x3, 0x0, 0x2E) /* IVB+ */
238 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_PS    OP_3D_MEDIA(0x3, 0x0, 0x2F) /* IVB+ */
239 #define OP_3DSTATE_URB_VS                       OP_3D_MEDIA(0x3, 0x0, 0x30) /* IVB+ */
240 #define OP_3DSTATE_URB_HS                       OP_3D_MEDIA(0x3, 0x0, 0x31) /* IVB+ */
241 #define OP_3DSTATE_URB_DS                       OP_3D_MEDIA(0x3, 0x0, 0x32) /* IVB+ */
242 #define OP_3DSTATE_URB_GS                       OP_3D_MEDIA(0x3, 0x0, 0x33) /* IVB+ */
243 #define OP_3DSTATE_GATHER_CONSTANT_VS           OP_3D_MEDIA(0x3, 0x0, 0x34) /* HSW+ */
244 #define OP_3DSTATE_GATHER_CONSTANT_GS           OP_3D_MEDIA(0x3, 0x0, 0x35) /* HSW+ */
245 #define OP_3DSTATE_GATHER_CONSTANT_HS           OP_3D_MEDIA(0x3, 0x0, 0x36) /* HSW+ */
246 #define OP_3DSTATE_GATHER_CONSTANT_DS           OP_3D_MEDIA(0x3, 0x0, 0x37) /* HSW+ */
247 #define OP_3DSTATE_GATHER_CONSTANT_PS           OP_3D_MEDIA(0x3, 0x0, 0x38) /* HSW+ */
248 #define OP_3DSTATE_DX9_CONSTANTF_VS             OP_3D_MEDIA(0x3, 0x0, 0x39) /* HSW+ */
249 #define OP_3DSTATE_DX9_CONSTANTF_PS             OP_3D_MEDIA(0x3, 0x0, 0x3A) /* HSW+ */
250 #define OP_3DSTATE_DX9_CONSTANTI_VS             OP_3D_MEDIA(0x3, 0x0, 0x3B) /* HSW+ */
251 #define OP_3DSTATE_DX9_CONSTANTI_PS             OP_3D_MEDIA(0x3, 0x0, 0x3C) /* HSW+ */
252 #define OP_3DSTATE_DX9_CONSTANTB_VS             OP_3D_MEDIA(0x3, 0x0, 0x3D) /* HSW+ */
253 #define OP_3DSTATE_DX9_CONSTANTB_PS             OP_3D_MEDIA(0x3, 0x0, 0x3E) /* HSW+ */
254 #define OP_3DSTATE_DX9_LOCAL_VALID_VS           OP_3D_MEDIA(0x3, 0x0, 0x3F) /* HSW+ */
255 #define OP_3DSTATE_DX9_LOCAL_VALID_PS           OP_3D_MEDIA(0x3, 0x0, 0x40) /* HSW+ */
256 #define OP_3DSTATE_DX9_GENERATE_ACTIVE_VS       OP_3D_MEDIA(0x3, 0x0, 0x41) /* HSW+ */
257 #define OP_3DSTATE_DX9_GENERATE_ACTIVE_PS       OP_3D_MEDIA(0x3, 0x0, 0x42) /* HSW+ */
258 #define OP_3DSTATE_BINDING_TABLE_EDIT_VS        OP_3D_MEDIA(0x3, 0x0, 0x43) /* HSW+ */
259 #define OP_3DSTATE_BINDING_TABLE_EDIT_GS        OP_3D_MEDIA(0x3, 0x0, 0x44) /* HSW+ */
260 #define OP_3DSTATE_BINDING_TABLE_EDIT_HS        OP_3D_MEDIA(0x3, 0x0, 0x45) /* HSW+ */
261 #define OP_3DSTATE_BINDING_TABLE_EDIT_DS        OP_3D_MEDIA(0x3, 0x0, 0x46) /* HSW+ */
262 #define OP_3DSTATE_BINDING_TABLE_EDIT_PS        OP_3D_MEDIA(0x3, 0x0, 0x47) /* HSW+ */
263 
264 #define OP_3DSTATE_VF_INSTANCING 		OP_3D_MEDIA(0x3, 0x0, 0x49) /* BDW+ */
265 #define OP_3DSTATE_VF_SGVS  			OP_3D_MEDIA(0x3, 0x0, 0x4A) /* BDW+ */
266 #define OP_3DSTATE_VF_TOPOLOGY   		OP_3D_MEDIA(0x3, 0x0, 0x4B) /* BDW+ */
267 #define OP_3DSTATE_WM_CHROMAKEY   		OP_3D_MEDIA(0x3, 0x0, 0x4C) /* BDW+ */
268 #define OP_3DSTATE_PS_BLEND   			OP_3D_MEDIA(0x3, 0x0, 0x4D) /* BDW+ */
269 #define OP_3DSTATE_WM_DEPTH_STENCIL   		OP_3D_MEDIA(0x3, 0x0, 0x4E) /* BDW+ */
270 #define OP_3DSTATE_PS_EXTRA   			OP_3D_MEDIA(0x3, 0x0, 0x4F) /* BDW+ */
271 #define OP_3DSTATE_RASTER   			OP_3D_MEDIA(0x3, 0x0, 0x50) /* BDW+ */
272 #define OP_3DSTATE_SBE_SWIZ   			OP_3D_MEDIA(0x3, 0x0, 0x51) /* BDW+ */
273 #define OP_3DSTATE_WM_HZ_OP   			OP_3D_MEDIA(0x3, 0x0, 0x52) /* BDW+ */
274 #define OP_3DSTATE_COMPONENT_PACKING		OP_3D_MEDIA(0x3, 0x0, 0x55) /* SKL+ */
275 
276 #define OP_3DSTATE_DRAWING_RECTANGLE            OP_3D_MEDIA(0x3, 0x1, 0x00)
277 #define OP_3DSTATE_SAMPLER_PALETTE_LOAD0        OP_3D_MEDIA(0x3, 0x1, 0x02)
278 #define OP_3DSTATE_CHROMA_KEY                   OP_3D_MEDIA(0x3, 0x1, 0x04)
279 #define OP_SNB_3DSTATE_DEPTH_BUFFER             OP_3D_MEDIA(0x3, 0x1, 0x05)
280 #define OP_3DSTATE_POLY_STIPPLE_OFFSET          OP_3D_MEDIA(0x3, 0x1, 0x06)
281 #define OP_3DSTATE_POLY_STIPPLE_PATTERN         OP_3D_MEDIA(0x3, 0x1, 0x07)
282 #define OP_3DSTATE_LINE_STIPPLE                 OP_3D_MEDIA(0x3, 0x1, 0x08)
283 #define OP_3DSTATE_AA_LINE_PARAMS               OP_3D_MEDIA(0x3, 0x1, 0x0A)
284 #define OP_3DSTATE_GS_SVB_INDEX                 OP_3D_MEDIA(0x3, 0x1, 0x0B)
285 #define OP_3DSTATE_SAMPLER_PALETTE_LOAD1        OP_3D_MEDIA(0x3, 0x1, 0x0C)
286 #define OP_3DSTATE_MULTISAMPLE_BDW		OP_3D_MEDIA(0x3, 0x0, 0x0D)
287 #define OP_SNB_3DSTATE_STENCIL_BUFFER           OP_3D_MEDIA(0x3, 0x1, 0x0E)
288 #define OP_SNB_3DSTATE_HIER_DEPTH_BUFFER        OP_3D_MEDIA(0x3, 0x1, 0x0F)
289 #define OP_SNB_3DSTATE_CLEAR_PARAMS             OP_3D_MEDIA(0x3, 0x1, 0x10)
290 #define OP_3DSTATE_MONOFILTER_SIZE              OP_3D_MEDIA(0x3, 0x1, 0x11)
291 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS       OP_3D_MEDIA(0x3, 0x1, 0x12) /* IVB+ */
292 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS       OP_3D_MEDIA(0x3, 0x1, 0x13) /* IVB+ */
293 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS       OP_3D_MEDIA(0x3, 0x1, 0x14) /* IVB+ */
294 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS       OP_3D_MEDIA(0x3, 0x1, 0x15) /* IVB+ */
295 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS       OP_3D_MEDIA(0x3, 0x1, 0x16) /* IVB+ */
296 #define OP_3DSTATE_SO_DECL_LIST                 OP_3D_MEDIA(0x3, 0x1, 0x17)
297 #define OP_3DSTATE_SO_BUFFER                    OP_3D_MEDIA(0x3, 0x1, 0x18)
298 #define OP_3DSTATE_BINDING_TABLE_POOL_ALLOC     OP_3D_MEDIA(0x3, 0x1, 0x19) /* HSW+ */
299 #define OP_3DSTATE_GATHER_POOL_ALLOC            OP_3D_MEDIA(0x3, 0x1, 0x1A) /* HSW+ */
300 #define OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1B) /* HSW+ */
301 #define OP_3DSTATE_SAMPLE_PATTERN               OP_3D_MEDIA(0x3, 0x1, 0x1C)
302 #define OP_PIPE_CONTROL                         OP_3D_MEDIA(0x3, 0x2, 0x00)
303 #define OP_3DPRIMITIVE                          OP_3D_MEDIA(0x3, 0x3, 0x00)
304 
305 /* VCCP Command Parser */
306 
307 /*
308  * Below MFX and VBE cmd definition is from vaapi intel driver project (BSD License)
309  * git://anongit.freedesktop.org/vaapi/intel-driver
310  * src/i965_defines.h
311  *
312  */
313 
314 #define OP_MFX(pipeline, op, sub_opa, sub_opb)     \
315 	(3 << 13 | \
316 	 (pipeline) << 11 | \
317 	 (op) << 8 | \
318 	 (sub_opa) << 5 | \
319 	 (sub_opb))
320 
321 #define OP_MFX_PIPE_MODE_SELECT                    OP_MFX(2, 0, 0, 0)  /* ALL */
322 #define OP_MFX_SURFACE_STATE                       OP_MFX(2, 0, 0, 1)  /* ALL */
323 #define OP_MFX_PIPE_BUF_ADDR_STATE                 OP_MFX(2, 0, 0, 2)  /* ALL */
324 #define OP_MFX_IND_OBJ_BASE_ADDR_STATE             OP_MFX(2, 0, 0, 3)  /* ALL */
325 #define OP_MFX_BSP_BUF_BASE_ADDR_STATE             OP_MFX(2, 0, 0, 4)  /* ALL */
326 #define OP_2_0_0_5                                 OP_MFX(2, 0, 0, 5)  /* ALL */
327 #define OP_MFX_STATE_POINTER                       OP_MFX(2, 0, 0, 6)  /* ALL */
328 #define OP_MFX_QM_STATE                            OP_MFX(2, 0, 0, 7)  /* IVB+ */
329 #define OP_MFX_FQM_STATE                           OP_MFX(2, 0, 0, 8)  /* IVB+ */
330 #define OP_MFX_PAK_INSERT_OBJECT                   OP_MFX(2, 0, 2, 8)  /* IVB+ */
331 #define OP_MFX_STITCH_OBJECT                       OP_MFX(2, 0, 2, 0xA)  /* IVB+ */
332 
333 #define OP_MFD_IT_OBJECT                           OP_MFX(2, 0, 1, 9) /* ALL */
334 
335 #define OP_MFX_WAIT                                OP_MFX(1, 0, 0, 0) /* IVB+ */
336 #define OP_MFX_AVC_IMG_STATE                       OP_MFX(2, 1, 0, 0) /* ALL */
337 #define OP_MFX_AVC_QM_STATE                        OP_MFX(2, 1, 0, 1) /* ALL */
338 #define OP_MFX_AVC_DIRECTMODE_STATE                OP_MFX(2, 1, 0, 2) /* ALL */
339 #define OP_MFX_AVC_SLICE_STATE                     OP_MFX(2, 1, 0, 3) /* ALL */
340 #define OP_MFX_AVC_REF_IDX_STATE                   OP_MFX(2, 1, 0, 4) /* ALL */
341 #define OP_MFX_AVC_WEIGHTOFFSET_STATE              OP_MFX(2, 1, 0, 5) /* ALL */
342 #define OP_MFD_AVC_PICID_STATE                     OP_MFX(2, 1, 1, 5) /* HSW+ */
343 #define OP_MFD_AVC_DPB_STATE			   OP_MFX(2, 1, 1, 6) /* IVB+ */
344 #define OP_MFD_AVC_SLICEADDR                       OP_MFX(2, 1, 1, 7) /* IVB+ */
345 #define OP_MFD_AVC_BSD_OBJECT                      OP_MFX(2, 1, 1, 8) /* ALL */
346 #define OP_MFC_AVC_PAK_OBJECT                      OP_MFX(2, 1, 2, 9) /* ALL */
347 
348 #define OP_MFX_VC1_PRED_PIPE_STATE                 OP_MFX(2, 2, 0, 1) /* ALL */
349 #define OP_MFX_VC1_DIRECTMODE_STATE                OP_MFX(2, 2, 0, 2) /* ALL */
350 #define OP_MFD_VC1_SHORT_PIC_STATE                 OP_MFX(2, 2, 1, 0) /* IVB+ */
351 #define OP_MFD_VC1_LONG_PIC_STATE                  OP_MFX(2, 2, 1, 1) /* IVB+ */
352 #define OP_MFD_VC1_BSD_OBJECT                      OP_MFX(2, 2, 1, 8) /* ALL */
353 
354 #define OP_MFX_MPEG2_PIC_STATE                     OP_MFX(2, 3, 0, 0) /* ALL */
355 #define OP_MFX_MPEG2_QM_STATE                      OP_MFX(2, 3, 0, 1) /* ALL */
356 #define OP_MFD_MPEG2_BSD_OBJECT                    OP_MFX(2, 3, 1, 8) /* ALL */
357 #define OP_MFC_MPEG2_SLICEGROUP_STATE              OP_MFX(2, 3, 2, 3) /* ALL */
358 #define OP_MFC_MPEG2_PAK_OBJECT                    OP_MFX(2, 3, 2, 9) /* ALL */
359 
360 #define OP_MFX_2_6_0_0                             OP_MFX(2, 6, 0, 0) /* IVB+ */
361 #define OP_MFX_2_6_0_8                             OP_MFX(2, 6, 0, 8) /* IVB+ */
362 #define OP_MFX_2_6_0_9                             OP_MFX(2, 6, 0, 9) /* IVB+ */
363 
364 #define OP_MFX_JPEG_PIC_STATE                      OP_MFX(2, 7, 0, 0)
365 #define OP_MFX_JPEG_HUFF_TABLE_STATE               OP_MFX(2, 7, 0, 2)
366 #define OP_MFD_JPEG_BSD_OBJECT                     OP_MFX(2, 7, 1, 8)
367 
368 #define OP_VEB(pipeline, op, sub_opa, sub_opb) \
369 	(3 << 13 | \
370 	 (pipeline) << 11 | \
371 	 (op) << 8 | \
372 	 (sub_opa) << 5 | \
373 	 (sub_opb))
374 
375 #define OP_VEB_SURFACE_STATE                       OP_VEB(2, 4, 0, 0)
376 #define OP_VEB_STATE                               OP_VEB(2, 4, 0, 2)
377 #define OP_VEB_DNDI_IECP_STATE                     OP_VEB(2, 4, 0, 3)
378 
379 struct parser_exec_state;
380 
381 typedef int (*parser_cmd_handler)(struct parser_exec_state *s);
382 
383 #define GVT_CMD_HASH_BITS   7
384 
385 /* which DWords need address fix */
386 #define ADDR_FIX_1(x1)			(1 << (x1))
387 #define ADDR_FIX_2(x1, x2)		(ADDR_FIX_1(x1) | ADDR_FIX_1(x2))
388 #define ADDR_FIX_3(x1, x2, x3)		(ADDR_FIX_1(x1) | ADDR_FIX_2(x2, x3))
389 #define ADDR_FIX_4(x1, x2, x3, x4)	(ADDR_FIX_1(x1) | ADDR_FIX_3(x2, x3, x4))
390 #define ADDR_FIX_5(x1, x2, x3, x4, x5)  (ADDR_FIX_1(x1) | ADDR_FIX_4(x2, x3, x4, x5))
391 
392 #define DWORD_FIELD(dword, end, start) \
393 	FIELD_GET(GENMASK(end, start), cmd_val(s, dword))
394 
395 #define OP_LENGTH_BIAS 2
396 #define CMD_LEN(value)  (value + OP_LENGTH_BIAS)
397 
gvt_check_valid_cmd_length(int len,int valid_len)398 static int gvt_check_valid_cmd_length(int len, int valid_len)
399 {
400 	if (valid_len != len) {
401 		gvt_err("len is not valid:  len=%u  valid_len=%u\n",
402 			len, valid_len);
403 		return -EFAULT;
404 	}
405 	return 0;
406 }
407 
408 struct cmd_info {
409 	const char *name;
410 	u32 opcode;
411 
412 #define F_LEN_MASK	3U
413 #define F_LEN_CONST  1U
414 #define F_LEN_VAR    0U
415 /* value is const although LEN maybe variable */
416 #define F_LEN_VAR_FIXED    (1<<1)
417 
418 /*
419  * command has its own ip advance logic
420  * e.g. MI_BATCH_START, MI_BATCH_END
421  */
422 #define F_IP_ADVANCE_CUSTOM (1<<2)
423 	u32 flag;
424 
425 #define R_RCS	BIT(RCS0)
426 #define R_VCS1  BIT(VCS0)
427 #define R_VCS2  BIT(VCS1)
428 #define R_VCS	(R_VCS1 | R_VCS2)
429 #define R_BCS	BIT(BCS0)
430 #define R_VECS	BIT(VECS0)
431 #define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS)
432 	/* rings that support this cmd: BLT/RCS/VCS/VECS */
433 	intel_engine_mask_t rings;
434 
435 	/* devices that support this cmd: SNB/IVB/HSW/... */
436 	u16 devices;
437 
438 	/* which DWords are address that need fix up.
439 	 * bit 0 means a 32-bit non address operand in command
440 	 * bit 1 means address operand, which could be 32-bit
441 	 * or 64-bit depending on different architectures.(
442 	 * defined by "gmadr_bytes_in_cmd" in intel_gvt.
443 	 * No matter the address length, each address only takes
444 	 * one bit in the bitmap.
445 	 */
446 	u16 addr_bitmap;
447 
448 	/* flag == F_LEN_CONST : command length
449 	 * flag == F_LEN_VAR : length bias bits
450 	 * Note: length is in DWord
451 	 */
452 	u32 len;
453 
454 	parser_cmd_handler handler;
455 
456 	/* valid length in DWord */
457 	u32 valid_len;
458 };
459 
460 struct cmd_entry {
461 	struct hlist_node hlist;
462 	const struct cmd_info *info;
463 };
464 
465 enum {
466 	RING_BUFFER_INSTRUCTION,
467 	BATCH_BUFFER_INSTRUCTION,
468 	BATCH_BUFFER_2ND_LEVEL,
469 	RING_BUFFER_CTX,
470 };
471 
472 enum {
473 	GTT_BUFFER,
474 	PPGTT_BUFFER
475 };
476 
477 struct parser_exec_state {
478 	struct intel_vgpu *vgpu;
479 	const struct intel_engine_cs *engine;
480 
481 	int buf_type;
482 
483 	/* batch buffer address type */
484 	int buf_addr_type;
485 
486 	/* graphics memory address of ring buffer start */
487 	unsigned long ring_start;
488 	unsigned long ring_size;
489 	unsigned long ring_head;
490 	unsigned long ring_tail;
491 
492 	/* instruction graphics memory address */
493 	unsigned long ip_gma;
494 
495 	/* mapped va of the instr_gma */
496 	void *ip_va;
497 	void *rb_va;
498 
499 	void *ret_bb_va;
500 	/* next instruction when return from  batch buffer to ring buffer */
501 	unsigned long ret_ip_gma_ring;
502 
503 	/* next instruction when return from 2nd batch buffer to batch buffer */
504 	unsigned long ret_ip_gma_bb;
505 
506 	/* batch buffer address type (GTT or PPGTT)
507 	 * used when ret from 2nd level batch buffer
508 	 */
509 	int saved_buf_addr_type;
510 	bool is_ctx_wa;
511 	bool is_init_ctx;
512 
513 	const struct cmd_info *info;
514 
515 	struct intel_vgpu_workload *workload;
516 };
517 
518 #define gmadr_dw_number(s)	\
519 	(s->vgpu->gvt->device_info.gmadr_bytes_in_cmd >> 2)
520 
521 static unsigned long bypass_scan_mask = 0;
522 
523 /* ring ALL, type = 0 */
524 static const struct sub_op_bits sub_op_mi[] = {
525 	{31, 29},
526 	{28, 23},
527 };
528 
529 static const struct decode_info decode_info_mi = {
530 	"MI",
531 	OP_LEN_MI,
532 	ARRAY_SIZE(sub_op_mi),
533 	sub_op_mi,
534 };
535 
536 /* ring RCS, command type 2 */
537 static const struct sub_op_bits sub_op_2d[] = {
538 	{31, 29},
539 	{28, 22},
540 };
541 
542 static const struct decode_info decode_info_2d = {
543 	"2D",
544 	OP_LEN_2D,
545 	ARRAY_SIZE(sub_op_2d),
546 	sub_op_2d,
547 };
548 
549 /* ring RCS, command type 3 */
550 static const struct sub_op_bits sub_op_3d_media[] = {
551 	{31, 29},
552 	{28, 27},
553 	{26, 24},
554 	{23, 16},
555 };
556 
557 static const struct decode_info decode_info_3d_media = {
558 	"3D_Media",
559 	OP_LEN_3D_MEDIA,
560 	ARRAY_SIZE(sub_op_3d_media),
561 	sub_op_3d_media,
562 };
563 
564 /* ring VCS, command type 3 */
565 static const struct sub_op_bits sub_op_mfx_vc[] = {
566 	{31, 29},
567 	{28, 27},
568 	{26, 24},
569 	{23, 21},
570 	{20, 16},
571 };
572 
573 static const struct decode_info decode_info_mfx_vc = {
574 	"MFX_VC",
575 	OP_LEN_MFX_VC,
576 	ARRAY_SIZE(sub_op_mfx_vc),
577 	sub_op_mfx_vc,
578 };
579 
580 /* ring VECS, command type 3 */
581 static const struct sub_op_bits sub_op_vebox[] = {
582 	{31, 29},
583 	{28, 27},
584 	{26, 24},
585 	{23, 21},
586 	{20, 16},
587 };
588 
589 static const struct decode_info decode_info_vebox = {
590 	"VEBOX",
591 	OP_LEN_VEBOX,
592 	ARRAY_SIZE(sub_op_vebox),
593 	sub_op_vebox,
594 };
595 
596 static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
597 	[RCS0] = {
598 		&decode_info_mi,
599 		NULL,
600 		NULL,
601 		&decode_info_3d_media,
602 		NULL,
603 		NULL,
604 		NULL,
605 		NULL,
606 	},
607 
608 	[VCS0] = {
609 		&decode_info_mi,
610 		NULL,
611 		NULL,
612 		&decode_info_mfx_vc,
613 		NULL,
614 		NULL,
615 		NULL,
616 		NULL,
617 	},
618 
619 	[BCS0] = {
620 		&decode_info_mi,
621 		NULL,
622 		&decode_info_2d,
623 		NULL,
624 		NULL,
625 		NULL,
626 		NULL,
627 		NULL,
628 	},
629 
630 	[VECS0] = {
631 		&decode_info_mi,
632 		NULL,
633 		NULL,
634 		&decode_info_vebox,
635 		NULL,
636 		NULL,
637 		NULL,
638 		NULL,
639 	},
640 
641 	[VCS1] = {
642 		&decode_info_mi,
643 		NULL,
644 		NULL,
645 		&decode_info_mfx_vc,
646 		NULL,
647 		NULL,
648 		NULL,
649 		NULL,
650 	},
651 };
652 
get_opcode(u32 cmd,const struct intel_engine_cs * engine)653 static inline u32 get_opcode(u32 cmd, const struct intel_engine_cs *engine)
654 {
655 	const struct decode_info *d_info;
656 
657 	d_info = ring_decode_info[engine->id][CMD_TYPE(cmd)];
658 	if (d_info == NULL)
659 		return INVALID_OP;
660 
661 	return cmd >> (32 - d_info->op_len);
662 }
663 
664 static inline const struct cmd_info *
find_cmd_entry(struct intel_gvt * gvt,unsigned int opcode,const struct intel_engine_cs * engine)665 find_cmd_entry(struct intel_gvt *gvt, unsigned int opcode,
666 	       const struct intel_engine_cs *engine)
667 {
668 	struct cmd_entry *e;
669 
670 	hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) {
671 		if (opcode == e->info->opcode &&
672 		    e->info->rings & engine->mask)
673 			return e->info;
674 	}
675 	return NULL;
676 }
677 
678 static inline const struct cmd_info *
get_cmd_info(struct intel_gvt * gvt,u32 cmd,const struct intel_engine_cs * engine)679 get_cmd_info(struct intel_gvt *gvt, u32 cmd,
680 	     const struct intel_engine_cs *engine)
681 {
682 	u32 opcode;
683 
684 	opcode = get_opcode(cmd, engine);
685 	if (opcode == INVALID_OP)
686 		return NULL;
687 
688 	return find_cmd_entry(gvt, opcode, engine);
689 }
690 
sub_op_val(u32 cmd,u32 hi,u32 low)691 static inline u32 sub_op_val(u32 cmd, u32 hi, u32 low)
692 {
693 	return (cmd >> low) & ((1U << (hi - low + 1)) - 1);
694 }
695 
print_opcode(u32 cmd,const struct intel_engine_cs * engine)696 static inline void print_opcode(u32 cmd, const struct intel_engine_cs *engine)
697 {
698 	const struct decode_info *d_info;
699 	int i;
700 
701 	d_info = ring_decode_info[engine->id][CMD_TYPE(cmd)];
702 	if (d_info == NULL)
703 		return;
704 
705 	gvt_dbg_cmd("opcode=0x%x %s sub_ops:",
706 			cmd >> (32 - d_info->op_len), d_info->name);
707 
708 	for (i = 0; i < d_info->nr_sub_op; i++)
709 		pr_err("0x%x ", sub_op_val(cmd, d_info->sub_op[i].hi,
710 					d_info->sub_op[i].low));
711 
712 	pr_err("\n");
713 }
714 
cmd_ptr(struct parser_exec_state * s,int index)715 static inline u32 *cmd_ptr(struct parser_exec_state *s, int index)
716 {
717 	return s->ip_va + (index << 2);
718 }
719 
cmd_val(struct parser_exec_state * s,int index)720 static inline u32 cmd_val(struct parser_exec_state *s, int index)
721 {
722 	return *cmd_ptr(s, index);
723 }
724 
is_init_ctx(struct parser_exec_state * s)725 static inline bool is_init_ctx(struct parser_exec_state *s)
726 {
727 	return (s->buf_type == RING_BUFFER_CTX && s->is_init_ctx);
728 }
729 
parser_exec_state_dump(struct parser_exec_state * s)730 static void parser_exec_state_dump(struct parser_exec_state *s)
731 {
732 	int cnt = 0;
733 	int i;
734 
735 	gvt_dbg_cmd("  vgpu%d RING%s: ring_start(%08lx) ring_end(%08lx)"
736 		    " ring_head(%08lx) ring_tail(%08lx)\n",
737 		    s->vgpu->id, s->engine->name,
738 		    s->ring_start, s->ring_start + s->ring_size,
739 		    s->ring_head, s->ring_tail);
740 
741 	gvt_dbg_cmd("  %s %s ip_gma(%08lx) ",
742 			s->buf_type == RING_BUFFER_INSTRUCTION ?
743 			"RING_BUFFER" : ((s->buf_type == RING_BUFFER_CTX) ?
744 				"CTX_BUFFER" : "BATCH_BUFFER"),
745 			s->buf_addr_type == GTT_BUFFER ?
746 			"GTT" : "PPGTT", s->ip_gma);
747 
748 	if (s->ip_va == NULL) {
749 		gvt_dbg_cmd(" ip_va(NULL)");
750 		return;
751 	}
752 
753 	gvt_dbg_cmd("  ip_va=%p: %08x %08x %08x %08x\n",
754 			s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
755 			cmd_val(s, 2), cmd_val(s, 3));
756 
757 	print_opcode(cmd_val(s, 0), s->engine);
758 
759 	s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12);
760 
761 	while (cnt < 1024) {
762 		gvt_dbg_cmd("ip_va=%p: ", s->ip_va);
763 		for (i = 0; i < 8; i++)
764 			gvt_dbg_cmd("%08x ", cmd_val(s, i));
765 		gvt_dbg_cmd("\n");
766 
767 		s->ip_va += 8 * sizeof(u32);
768 		cnt += 8;
769 	}
770 }
771 
update_ip_va(struct parser_exec_state * s)772 static inline void update_ip_va(struct parser_exec_state *s)
773 {
774 	unsigned long len = 0;
775 
776 	if (WARN_ON(s->ring_head == s->ring_tail))
777 		return;
778 
779 	if (s->buf_type == RING_BUFFER_INSTRUCTION ||
780 			s->buf_type == RING_BUFFER_CTX) {
781 		unsigned long ring_top = s->ring_start + s->ring_size;
782 
783 		if (s->ring_head > s->ring_tail) {
784 			if (s->ip_gma >= s->ring_head && s->ip_gma < ring_top)
785 				len = (s->ip_gma - s->ring_head);
786 			else if (s->ip_gma >= s->ring_start &&
787 					s->ip_gma <= s->ring_tail)
788 				len = (ring_top - s->ring_head) +
789 					(s->ip_gma - s->ring_start);
790 		} else
791 			len = (s->ip_gma - s->ring_head);
792 
793 		s->ip_va = s->rb_va + len;
794 	} else {/* shadow batch buffer */
795 		s->ip_va = s->ret_bb_va;
796 	}
797 }
798 
ip_gma_set(struct parser_exec_state * s,unsigned long ip_gma)799 static inline int ip_gma_set(struct parser_exec_state *s,
800 		unsigned long ip_gma)
801 {
802 	WARN_ON(!IS_ALIGNED(ip_gma, 4));
803 
804 	s->ip_gma = ip_gma;
805 	update_ip_va(s);
806 	return 0;
807 }
808 
ip_gma_advance(struct parser_exec_state * s,unsigned int dw_len)809 static inline int ip_gma_advance(struct parser_exec_state *s,
810 		unsigned int dw_len)
811 {
812 	s->ip_gma += (dw_len << 2);
813 
814 	if (s->buf_type == RING_BUFFER_INSTRUCTION) {
815 		if (s->ip_gma >= s->ring_start + s->ring_size)
816 			s->ip_gma -= s->ring_size;
817 		update_ip_va(s);
818 	} else {
819 		s->ip_va += (dw_len << 2);
820 	}
821 
822 	return 0;
823 }
824 
get_cmd_length(const struct cmd_info * info,u32 cmd)825 static inline int get_cmd_length(const struct cmd_info *info, u32 cmd)
826 {
827 	if ((info->flag & F_LEN_MASK) == F_LEN_CONST)
828 		return info->len;
829 	else
830 		return (cmd & ((1U << info->len) - 1)) + 2;
831 	return 0;
832 }
833 
cmd_length(struct parser_exec_state * s)834 static inline int cmd_length(struct parser_exec_state *s)
835 {
836 	return get_cmd_length(s->info, cmd_val(s, 0));
837 }
838 
839 /* do not remove this, some platform may need clflush here */
840 #define patch_value(s, addr, val) do { \
841 	*addr = val; \
842 } while (0)
843 
is_mocs_mmio(unsigned int offset)844 static inline bool is_mocs_mmio(unsigned int offset)
845 {
846 	return ((offset >= 0xc800) && (offset <= 0xcff8)) ||
847 		((offset >= 0xb020) && (offset <= 0xb0a0));
848 }
849 
is_cmd_update_pdps(unsigned int offset,struct parser_exec_state * s)850 static int is_cmd_update_pdps(unsigned int offset,
851 			      struct parser_exec_state *s)
852 {
853 	u32 base = s->workload->engine->mmio_base;
854 	return i915_mmio_reg_equal(_MMIO(offset), GEN8_RING_PDP_UDW(base, 0));
855 }
856 
cmd_pdp_mmio_update_handler(struct parser_exec_state * s,unsigned int offset,unsigned int index)857 static int cmd_pdp_mmio_update_handler(struct parser_exec_state *s,
858 				       unsigned int offset, unsigned int index)
859 {
860 	struct intel_vgpu *vgpu = s->vgpu;
861 	struct intel_vgpu_mm *shadow_mm = s->workload->shadow_mm;
862 	struct intel_vgpu_mm *mm;
863 	u64 pdps[GEN8_3LVL_PDPES];
864 
865 	if (shadow_mm->ppgtt_mm.root_entry_type ==
866 	    GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
867 		pdps[0] = (u64)cmd_val(s, 2) << 32;
868 		pdps[0] |= cmd_val(s, 4);
869 
870 		mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps);
871 		if (!mm) {
872 			gvt_vgpu_err("failed to get the 4-level shadow vm\n");
873 			return -EINVAL;
874 		}
875 		intel_vgpu_mm_get(mm);
876 		list_add_tail(&mm->ppgtt_mm.link,
877 			      &s->workload->lri_shadow_mm);
878 		*cmd_ptr(s, 2) = upper_32_bits(mm->ppgtt_mm.shadow_pdps[0]);
879 		*cmd_ptr(s, 4) = lower_32_bits(mm->ppgtt_mm.shadow_pdps[0]);
880 	} else {
881 		/* Currently all guests use PML4 table and now can't
882 		 * have a guest with 3-level table but uses LRI for
883 		 * PPGTT update. So this is simply un-testable. */
884 		GEM_BUG_ON(1);
885 		gvt_vgpu_err("invalid shared shadow vm type\n");
886 		return -EINVAL;
887 	}
888 	return 0;
889 }
890 
cmd_reg_handler(struct parser_exec_state * s,unsigned int offset,unsigned int index,char * cmd)891 static int cmd_reg_handler(struct parser_exec_state *s,
892 	unsigned int offset, unsigned int index, char *cmd)
893 {
894 	struct intel_vgpu *vgpu = s->vgpu;
895 	struct intel_gvt *gvt = vgpu->gvt;
896 	u32 ctx_sr_ctl;
897 	u32 *vreg, vreg_old;
898 
899 	if (offset + 4 > gvt->device_info.mmio_size) {
900 		gvt_vgpu_err("%s access to (%x) outside of MMIO range\n",
901 				cmd, offset);
902 		return -EFAULT;
903 	}
904 
905 	if (is_init_ctx(s)) {
906 		struct intel_gvt_mmio_info *mmio_info;
907 
908 		intel_gvt_mmio_set_cmd_accessible(gvt, offset);
909 		mmio_info = intel_gvt_find_mmio_info(gvt, offset);
910 		if (mmio_info && mmio_info->write)
911 			intel_gvt_mmio_set_cmd_write_patch(gvt, offset);
912 		return 0;
913 	}
914 
915 	if (!intel_gvt_mmio_is_cmd_accessible(gvt, offset)) {
916 		gvt_vgpu_err("%s access to non-render register (%x)\n",
917 				cmd, offset);
918 		return -EBADRQC;
919 	}
920 
921 	if (!strncmp(cmd, "srm", 3) ||
922 			!strncmp(cmd, "lrm", 3)) {
923 		if (offset == i915_mmio_reg_offset(GEN8_L3SQCREG4) ||
924 		    offset == 0x21f0 ||
925 		    (IS_BROADWELL(gvt->gt->i915) &&
926 		     offset == i915_mmio_reg_offset(INSTPM)))
927 			return 0;
928 		else {
929 			gvt_vgpu_err("%s access to register (%x)\n",
930 					cmd, offset);
931 			return -EPERM;
932 		}
933 	}
934 
935 	if (!strncmp(cmd, "lrr-src", 7) ||
936 			!strncmp(cmd, "lrr-dst", 7)) {
937 		if (IS_BROADWELL(gvt->gt->i915) && offset == 0x215c)
938 			return 0;
939 		else {
940 			gvt_vgpu_err("not allowed cmd %s reg (%x)\n", cmd, offset);
941 			return -EPERM;
942 		}
943 	}
944 
945 	if (!strncmp(cmd, "pipe_ctrl", 9)) {
946 		/* TODO: add LRI POST logic here */
947 		return 0;
948 	}
949 
950 	if (strncmp(cmd, "lri", 3))
951 		return -EPERM;
952 
953 	/* below are all lri handlers */
954 	vreg = &vgpu_vreg(s->vgpu, offset);
955 
956 	if (is_cmd_update_pdps(offset, s) &&
957 	    cmd_pdp_mmio_update_handler(s, offset, index))
958 		return -EINVAL;
959 
960 	if (offset == i915_mmio_reg_offset(DERRMR) ||
961 		offset == i915_mmio_reg_offset(FORCEWAKE_MT)) {
962 		/* Writing to HW VGT_PVINFO_PAGE offset will be discarded */
963 		patch_value(s, cmd_ptr(s, index), VGT_PVINFO_PAGE);
964 	}
965 
966 	if (is_mocs_mmio(offset))
967 		*vreg = cmd_val(s, index + 1);
968 
969 	vreg_old = *vreg;
970 
971 	if (intel_gvt_mmio_is_cmd_write_patch(gvt, offset)) {
972 		u32 cmdval_new, cmdval;
973 		struct intel_gvt_mmio_info *mmio_info;
974 
975 		cmdval = cmd_val(s, index + 1);
976 
977 		mmio_info = intel_gvt_find_mmio_info(gvt, offset);
978 		if (!mmio_info) {
979 			cmdval_new = cmdval;
980 		} else {
981 			u64 ro_mask = mmio_info->ro_mask;
982 			int ret;
983 
984 			if (likely(!ro_mask))
985 				ret = mmio_info->write(s->vgpu, offset,
986 						&cmdval, 4);
987 			else {
988 				gvt_vgpu_err("try to write RO reg %x\n",
989 						offset);
990 				ret = -EBADRQC;
991 			}
992 			if (ret)
993 				return ret;
994 			cmdval_new = *vreg;
995 		}
996 		if (cmdval_new != cmdval)
997 			patch_value(s, cmd_ptr(s, index+1), cmdval_new);
998 	}
999 
1000 	/* only patch cmd. restore vreg value if changed in mmio write handler*/
1001 	*vreg = vreg_old;
1002 
1003 	/* TODO
1004 	 * In order to let workload with inhibit context to generate
1005 	 * correct image data into memory, vregs values will be loaded to
1006 	 * hw via LRIs in the workload with inhibit context. But as
1007 	 * indirect context is loaded prior to LRIs in workload, we don't
1008 	 * want reg values specified in indirect context overwritten by
1009 	 * LRIs in workloads. So, when scanning an indirect context, we
1010 	 * update reg values in it into vregs, so LRIs in workload with
1011 	 * inhibit context will restore with correct values
1012 	 */
1013 	if (GRAPHICS_VER(s->engine->i915) == 9 &&
1014 	    intel_gvt_mmio_is_sr_in_ctx(gvt, offset) &&
1015 	    !strncmp(cmd, "lri", 3)) {
1016 		intel_gvt_read_gpa(s->vgpu,
1017 			s->workload->ring_context_gpa + 12, &ctx_sr_ctl, 4);
1018 		/* check inhibit context */
1019 		if (ctx_sr_ctl & 1) {
1020 			u32 data = cmd_val(s, index + 1);
1021 
1022 			if (intel_gvt_mmio_has_mode_mask(s->vgpu->gvt, offset))
1023 				intel_vgpu_mask_mmio_write(vgpu,
1024 							offset, &data, 4);
1025 			else
1026 				vgpu_vreg(vgpu, offset) = data;
1027 		}
1028 	}
1029 
1030 	return 0;
1031 }
1032 
1033 #define cmd_reg(s, i) \
1034 	(cmd_val(s, i) & GENMASK(22, 2))
1035 
1036 #define cmd_reg_inhibit(s, i) \
1037 	(cmd_val(s, i) & GENMASK(22, 18))
1038 
1039 #define cmd_gma(s, i) \
1040 	(cmd_val(s, i) & GENMASK(31, 2))
1041 
1042 #define cmd_gma_hi(s, i) \
1043 	(cmd_val(s, i) & GENMASK(15, 0))
1044 
cmd_handler_lri(struct parser_exec_state * s)1045 static int cmd_handler_lri(struct parser_exec_state *s)
1046 {
1047 	int i, ret = 0;
1048 	int cmd_len = cmd_length(s);
1049 
1050 	for (i = 1; i < cmd_len; i += 2) {
1051 		if (IS_BROADWELL(s->engine->i915) && s->engine->id != RCS0) {
1052 			if (s->engine->id == BCS0 &&
1053 			    cmd_reg(s, i) == i915_mmio_reg_offset(DERRMR))
1054 				ret |= 0;
1055 			else
1056 				ret |= cmd_reg_inhibit(s, i) ? -EBADRQC : 0;
1057 		}
1058 		if (ret)
1059 			break;
1060 		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri");
1061 		if (ret)
1062 			break;
1063 	}
1064 	return ret;
1065 }
1066 
cmd_handler_lrr(struct parser_exec_state * s)1067 static int cmd_handler_lrr(struct parser_exec_state *s)
1068 {
1069 	int i, ret = 0;
1070 	int cmd_len = cmd_length(s);
1071 
1072 	for (i = 1; i < cmd_len; i += 2) {
1073 		if (IS_BROADWELL(s->engine->i915))
1074 			ret |= ((cmd_reg_inhibit(s, i) ||
1075 				 (cmd_reg_inhibit(s, i + 1)))) ?
1076 				-EBADRQC : 0;
1077 		if (ret)
1078 			break;
1079 		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src");
1080 		if (ret)
1081 			break;
1082 		ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst");
1083 		if (ret)
1084 			break;
1085 	}
1086 	return ret;
1087 }
1088 
1089 static inline int cmd_address_audit(struct parser_exec_state *s,
1090 		unsigned long guest_gma, int op_size, bool index_mode);
1091 
cmd_handler_lrm(struct parser_exec_state * s)1092 static int cmd_handler_lrm(struct parser_exec_state *s)
1093 {
1094 	struct intel_gvt *gvt = s->vgpu->gvt;
1095 	int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
1096 	unsigned long gma;
1097 	int i, ret = 0;
1098 	int cmd_len = cmd_length(s);
1099 
1100 	for (i = 1; i < cmd_len;) {
1101 		if (IS_BROADWELL(s->engine->i915))
1102 			ret |= (cmd_reg_inhibit(s, i)) ? -EBADRQC : 0;
1103 		if (ret)
1104 			break;
1105 		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm");
1106 		if (ret)
1107 			break;
1108 		if (cmd_val(s, 0) & (1 << 22)) {
1109 			gma = cmd_gma(s, i + 1);
1110 			if (gmadr_bytes == 8)
1111 				gma |= (cmd_gma_hi(s, i + 2)) << 32;
1112 			ret |= cmd_address_audit(s, gma, sizeof(u32), false);
1113 			if (ret)
1114 				break;
1115 		}
1116 		i += gmadr_dw_number(s) + 1;
1117 	}
1118 	return ret;
1119 }
1120 
cmd_handler_srm(struct parser_exec_state * s)1121 static int cmd_handler_srm(struct parser_exec_state *s)
1122 {
1123 	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
1124 	unsigned long gma;
1125 	int i, ret = 0;
1126 	int cmd_len = cmd_length(s);
1127 
1128 	for (i = 1; i < cmd_len;) {
1129 		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm");
1130 		if (ret)
1131 			break;
1132 		if (cmd_val(s, 0) & (1 << 22)) {
1133 			gma = cmd_gma(s, i + 1);
1134 			if (gmadr_bytes == 8)
1135 				gma |= (cmd_gma_hi(s, i + 2)) << 32;
1136 			ret |= cmd_address_audit(s, gma, sizeof(u32), false);
1137 			if (ret)
1138 				break;
1139 		}
1140 		i += gmadr_dw_number(s) + 1;
1141 	}
1142 	return ret;
1143 }
1144 
1145 struct cmd_interrupt_event {
1146 	int pipe_control_notify;
1147 	int mi_flush_dw;
1148 	int mi_user_interrupt;
1149 };
1150 
1151 static const struct cmd_interrupt_event cmd_interrupt_events[] = {
1152 	[RCS0] = {
1153 		.pipe_control_notify = RCS_PIPE_CONTROL,
1154 		.mi_flush_dw = INTEL_GVT_EVENT_RESERVED,
1155 		.mi_user_interrupt = RCS_MI_USER_INTERRUPT,
1156 	},
1157 	[BCS0] = {
1158 		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
1159 		.mi_flush_dw = BCS_MI_FLUSH_DW,
1160 		.mi_user_interrupt = BCS_MI_USER_INTERRUPT,
1161 	},
1162 	[VCS0] = {
1163 		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
1164 		.mi_flush_dw = VCS_MI_FLUSH_DW,
1165 		.mi_user_interrupt = VCS_MI_USER_INTERRUPT,
1166 	},
1167 	[VCS1] = {
1168 		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
1169 		.mi_flush_dw = VCS2_MI_FLUSH_DW,
1170 		.mi_user_interrupt = VCS2_MI_USER_INTERRUPT,
1171 	},
1172 	[VECS0] = {
1173 		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
1174 		.mi_flush_dw = VECS_MI_FLUSH_DW,
1175 		.mi_user_interrupt = VECS_MI_USER_INTERRUPT,
1176 	},
1177 };
1178 
cmd_handler_pipe_control(struct parser_exec_state * s)1179 static int cmd_handler_pipe_control(struct parser_exec_state *s)
1180 {
1181 	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
1182 	unsigned long gma;
1183 	bool index_mode = false;
1184 	unsigned int post_sync;
1185 	int ret = 0;
1186 	u32 hws_pga, val;
1187 
1188 	post_sync = (cmd_val(s, 1) & PIPE_CONTROL_POST_SYNC_OP_MASK) >> 14;
1189 
1190 	/* LRI post sync */
1191 	if (cmd_val(s, 1) & PIPE_CONTROL_MMIO_WRITE)
1192 		ret = cmd_reg_handler(s, cmd_reg(s, 2), 1, "pipe_ctrl");
1193 	/* post sync */
1194 	else if (post_sync) {
1195 		if (post_sync == 2)
1196 			ret = cmd_reg_handler(s, 0x2350, 1, "pipe_ctrl");
1197 		else if (post_sync == 3)
1198 			ret = cmd_reg_handler(s, 0x2358, 1, "pipe_ctrl");
1199 		else if (post_sync == 1) {
1200 			/* check ggtt*/
1201 			if ((cmd_val(s, 1) & PIPE_CONTROL_GLOBAL_GTT_IVB)) {
1202 				gma = cmd_val(s, 2) & GENMASK(31, 3);
1203 				if (gmadr_bytes == 8)
1204 					gma |= (cmd_gma_hi(s, 3)) << 32;
1205 				/* Store Data Index */
1206 				if (cmd_val(s, 1) & (1 << 21))
1207 					index_mode = true;
1208 				ret |= cmd_address_audit(s, gma, sizeof(u64),
1209 						index_mode);
1210 				if (ret)
1211 					return ret;
1212 				if (index_mode) {
1213 					hws_pga = s->vgpu->hws_pga[s->engine->id];
1214 					gma = hws_pga + gma;
1215 					patch_value(s, cmd_ptr(s, 2), gma);
1216 					val = cmd_val(s, 1) & (~(1 << 21));
1217 					patch_value(s, cmd_ptr(s, 1), val);
1218 				}
1219 			}
1220 		}
1221 	}
1222 
1223 	if (ret)
1224 		return ret;
1225 
1226 	if (cmd_val(s, 1) & PIPE_CONTROL_NOTIFY)
1227 		set_bit(cmd_interrupt_events[s->engine->id].pipe_control_notify,
1228 			s->workload->pending_events);
1229 	return 0;
1230 }
1231 
cmd_handler_mi_user_interrupt(struct parser_exec_state * s)1232 static int cmd_handler_mi_user_interrupt(struct parser_exec_state *s)
1233 {
1234 	set_bit(cmd_interrupt_events[s->engine->id].mi_user_interrupt,
1235 		s->workload->pending_events);
1236 	patch_value(s, cmd_ptr(s, 0), MI_NOOP);
1237 	return 0;
1238 }
1239 
cmd_advance_default(struct parser_exec_state * s)1240 static int cmd_advance_default(struct parser_exec_state *s)
1241 {
1242 	return ip_gma_advance(s, cmd_length(s));
1243 }
1244 
cmd_handler_mi_batch_buffer_end(struct parser_exec_state * s)1245 static int cmd_handler_mi_batch_buffer_end(struct parser_exec_state *s)
1246 {
1247 	int ret;
1248 
1249 	if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
1250 		s->buf_type = BATCH_BUFFER_INSTRUCTION;
1251 		ret = ip_gma_set(s, s->ret_ip_gma_bb);
1252 		s->buf_addr_type = s->saved_buf_addr_type;
1253 	} else if (s->buf_type == RING_BUFFER_CTX) {
1254 		ret = ip_gma_set(s, s->ring_tail);
1255 	} else {
1256 		s->buf_type = RING_BUFFER_INSTRUCTION;
1257 		s->buf_addr_type = GTT_BUFFER;
1258 		if (s->ret_ip_gma_ring >= s->ring_start + s->ring_size)
1259 			s->ret_ip_gma_ring -= s->ring_size;
1260 		ret = ip_gma_set(s, s->ret_ip_gma_ring);
1261 	}
1262 	return ret;
1263 }
1264 
1265 struct mi_display_flip_command_info {
1266 	int pipe;
1267 	int plane;
1268 	int event;
1269 	i915_reg_t stride_reg;
1270 	i915_reg_t ctrl_reg;
1271 	i915_reg_t surf_reg;
1272 	u64 stride_val;
1273 	u64 tile_val;
1274 	u64 surf_val;
1275 	bool async_flip;
1276 };
1277 
1278 struct plane_code_mapping {
1279 	int pipe;
1280 	int plane;
1281 	int event;
1282 };
1283 
gen8_decode_mi_display_flip(struct parser_exec_state * s,struct mi_display_flip_command_info * info)1284 static int gen8_decode_mi_display_flip(struct parser_exec_state *s,
1285 		struct mi_display_flip_command_info *info)
1286 {
1287 	struct drm_i915_private *dev_priv = s->engine->i915;
1288 	struct plane_code_mapping gen8_plane_code[] = {
1289 		[0] = {PIPE_A, PLANE_A, PRIMARY_A_FLIP_DONE},
1290 		[1] = {PIPE_B, PLANE_A, PRIMARY_B_FLIP_DONE},
1291 		[2] = {PIPE_A, PLANE_B, SPRITE_A_FLIP_DONE},
1292 		[3] = {PIPE_B, PLANE_B, SPRITE_B_FLIP_DONE},
1293 		[4] = {PIPE_C, PLANE_A, PRIMARY_C_FLIP_DONE},
1294 		[5] = {PIPE_C, PLANE_B, SPRITE_C_FLIP_DONE},
1295 	};
1296 	u32 dword0, dword1, dword2;
1297 	u32 v;
1298 
1299 	dword0 = cmd_val(s, 0);
1300 	dword1 = cmd_val(s, 1);
1301 	dword2 = cmd_val(s, 2);
1302 
1303 	v = (dword0 & GENMASK(21, 19)) >> 19;
1304 	if (drm_WARN_ON(&dev_priv->drm, v >= ARRAY_SIZE(gen8_plane_code)))
1305 		return -EBADRQC;
1306 
1307 	info->pipe = gen8_plane_code[v].pipe;
1308 	info->plane = gen8_plane_code[v].plane;
1309 	info->event = gen8_plane_code[v].event;
1310 	info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
1311 	info->tile_val = (dword1 & 0x1);
1312 	info->surf_val = (dword2 & GENMASK(31, 12)) >> 12;
1313 	info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1);
1314 
1315 	if (info->plane == PLANE_A) {
1316 		info->ctrl_reg = DSPCNTR(info->pipe);
1317 		info->stride_reg = DSPSTRIDE(info->pipe);
1318 		info->surf_reg = DSPSURF(info->pipe);
1319 	} else if (info->plane == PLANE_B) {
1320 		info->ctrl_reg = SPRCTL(info->pipe);
1321 		info->stride_reg = SPRSTRIDE(info->pipe);
1322 		info->surf_reg = SPRSURF(info->pipe);
1323 	} else {
1324 		drm_WARN_ON(&dev_priv->drm, 1);
1325 		return -EBADRQC;
1326 	}
1327 	return 0;
1328 }
1329 
skl_decode_mi_display_flip(struct parser_exec_state * s,struct mi_display_flip_command_info * info)1330 static int skl_decode_mi_display_flip(struct parser_exec_state *s,
1331 		struct mi_display_flip_command_info *info)
1332 {
1333 	struct drm_i915_private *dev_priv = s->engine->i915;
1334 	struct intel_vgpu *vgpu = s->vgpu;
1335 	u32 dword0 = cmd_val(s, 0);
1336 	u32 dword1 = cmd_val(s, 1);
1337 	u32 dword2 = cmd_val(s, 2);
1338 	u32 plane = (dword0 & GENMASK(12, 8)) >> 8;
1339 
1340 	info->plane = PRIMARY_PLANE;
1341 
1342 	switch (plane) {
1343 	case MI_DISPLAY_FLIP_SKL_PLANE_1_A:
1344 		info->pipe = PIPE_A;
1345 		info->event = PRIMARY_A_FLIP_DONE;
1346 		break;
1347 	case MI_DISPLAY_FLIP_SKL_PLANE_1_B:
1348 		info->pipe = PIPE_B;
1349 		info->event = PRIMARY_B_FLIP_DONE;
1350 		break;
1351 	case MI_DISPLAY_FLIP_SKL_PLANE_1_C:
1352 		info->pipe = PIPE_C;
1353 		info->event = PRIMARY_C_FLIP_DONE;
1354 		break;
1355 
1356 	case MI_DISPLAY_FLIP_SKL_PLANE_2_A:
1357 		info->pipe = PIPE_A;
1358 		info->event = SPRITE_A_FLIP_DONE;
1359 		info->plane = SPRITE_PLANE;
1360 		break;
1361 	case MI_DISPLAY_FLIP_SKL_PLANE_2_B:
1362 		info->pipe = PIPE_B;
1363 		info->event = SPRITE_B_FLIP_DONE;
1364 		info->plane = SPRITE_PLANE;
1365 		break;
1366 	case MI_DISPLAY_FLIP_SKL_PLANE_2_C:
1367 		info->pipe = PIPE_C;
1368 		info->event = SPRITE_C_FLIP_DONE;
1369 		info->plane = SPRITE_PLANE;
1370 		break;
1371 
1372 	default:
1373 		gvt_vgpu_err("unknown plane code %d\n", plane);
1374 		return -EBADRQC;
1375 	}
1376 
1377 	info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
1378 	info->tile_val = (dword1 & GENMASK(2, 0));
1379 	info->surf_val = (dword2 & GENMASK(31, 12)) >> 12;
1380 	info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1);
1381 
1382 	info->ctrl_reg = DSPCNTR(info->pipe);
1383 	info->stride_reg = DSPSTRIDE(info->pipe);
1384 	info->surf_reg = DSPSURF(info->pipe);
1385 
1386 	return 0;
1387 }
1388 
gen8_check_mi_display_flip(struct parser_exec_state * s,struct mi_display_flip_command_info * info)1389 static int gen8_check_mi_display_flip(struct parser_exec_state *s,
1390 		struct mi_display_flip_command_info *info)
1391 {
1392 	u32 stride, tile;
1393 
1394 	if (!info->async_flip)
1395 		return 0;
1396 
1397 	if (GRAPHICS_VER(s->engine->i915) >= 9) {
1398 		stride = vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(9, 0);
1399 		tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) &
1400 				GENMASK(12, 10)) >> 10;
1401 	} else {
1402 		stride = (vgpu_vreg_t(s->vgpu, info->stride_reg) &
1403 				GENMASK(15, 6)) >> 6;
1404 		tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10;
1405 	}
1406 
1407 	if (stride != info->stride_val)
1408 		gvt_dbg_cmd("cannot change stride during async flip\n");
1409 
1410 	if (tile != info->tile_val)
1411 		gvt_dbg_cmd("cannot change tile during async flip\n");
1412 
1413 	return 0;
1414 }
1415 
gen8_update_plane_mmio_from_mi_display_flip(struct parser_exec_state * s,struct mi_display_flip_command_info * info)1416 static int gen8_update_plane_mmio_from_mi_display_flip(
1417 		struct parser_exec_state *s,
1418 		struct mi_display_flip_command_info *info)
1419 {
1420 	struct drm_i915_private *dev_priv = s->engine->i915;
1421 	struct intel_vgpu *vgpu = s->vgpu;
1422 
1423 	set_mask_bits(&vgpu_vreg_t(vgpu, info->surf_reg), GENMASK(31, 12),
1424 		      info->surf_val << 12);
1425 	if (GRAPHICS_VER(dev_priv) >= 9) {
1426 		set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(9, 0),
1427 			      info->stride_val);
1428 		set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(12, 10),
1429 			      info->tile_val << 10);
1430 	} else {
1431 		set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(15, 6),
1432 			      info->stride_val << 6);
1433 		set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(10, 10),
1434 			      info->tile_val << 10);
1435 	}
1436 
1437 	if (info->plane == PLANE_PRIMARY)
1438 		vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(info->pipe))++;
1439 
1440 	if (info->async_flip)
1441 		intel_vgpu_trigger_virtual_event(vgpu, info->event);
1442 	else
1443 		set_bit(info->event, vgpu->irq.flip_done_event[info->pipe]);
1444 
1445 	return 0;
1446 }
1447 
decode_mi_display_flip(struct parser_exec_state * s,struct mi_display_flip_command_info * info)1448 static int decode_mi_display_flip(struct parser_exec_state *s,
1449 		struct mi_display_flip_command_info *info)
1450 {
1451 	if (IS_BROADWELL(s->engine->i915))
1452 		return gen8_decode_mi_display_flip(s, info);
1453 	if (GRAPHICS_VER(s->engine->i915) >= 9)
1454 		return skl_decode_mi_display_flip(s, info);
1455 
1456 	return -ENODEV;
1457 }
1458 
check_mi_display_flip(struct parser_exec_state * s,struct mi_display_flip_command_info * info)1459 static int check_mi_display_flip(struct parser_exec_state *s,
1460 		struct mi_display_flip_command_info *info)
1461 {
1462 	return gen8_check_mi_display_flip(s, info);
1463 }
1464 
update_plane_mmio_from_mi_display_flip(struct parser_exec_state * s,struct mi_display_flip_command_info * info)1465 static int update_plane_mmio_from_mi_display_flip(
1466 		struct parser_exec_state *s,
1467 		struct mi_display_flip_command_info *info)
1468 {
1469 	return gen8_update_plane_mmio_from_mi_display_flip(s, info);
1470 }
1471 
cmd_handler_mi_display_flip(struct parser_exec_state * s)1472 static int cmd_handler_mi_display_flip(struct parser_exec_state *s)
1473 {
1474 	struct mi_display_flip_command_info info;
1475 	struct intel_vgpu *vgpu = s->vgpu;
1476 	int ret;
1477 	int i;
1478 	int len = cmd_length(s);
1479 	u32 valid_len = CMD_LEN(1);
1480 
1481 	/* Flip Type == Stereo 3D Flip */
1482 	if (DWORD_FIELD(2, 1, 0) == 2)
1483 		valid_len++;
1484 	ret = gvt_check_valid_cmd_length(cmd_length(s),
1485 			valid_len);
1486 	if (ret)
1487 		return ret;
1488 
1489 	ret = decode_mi_display_flip(s, &info);
1490 	if (ret) {
1491 		gvt_vgpu_err("fail to decode MI display flip command\n");
1492 		return ret;
1493 	}
1494 
1495 	ret = check_mi_display_flip(s, &info);
1496 	if (ret) {
1497 		gvt_vgpu_err("invalid MI display flip command\n");
1498 		return ret;
1499 	}
1500 
1501 	ret = update_plane_mmio_from_mi_display_flip(s, &info);
1502 	if (ret) {
1503 		gvt_vgpu_err("fail to update plane mmio\n");
1504 		return ret;
1505 	}
1506 
1507 	for (i = 0; i < len; i++)
1508 		patch_value(s, cmd_ptr(s, i), MI_NOOP);
1509 	return 0;
1510 }
1511 
is_wait_for_flip_pending(u32 cmd)1512 static bool is_wait_for_flip_pending(u32 cmd)
1513 {
1514 	return cmd & (MI_WAIT_FOR_PLANE_A_FLIP_PENDING |
1515 			MI_WAIT_FOR_PLANE_B_FLIP_PENDING |
1516 			MI_WAIT_FOR_PLANE_C_FLIP_PENDING |
1517 			MI_WAIT_FOR_SPRITE_A_FLIP_PENDING |
1518 			MI_WAIT_FOR_SPRITE_B_FLIP_PENDING |
1519 			MI_WAIT_FOR_SPRITE_C_FLIP_PENDING);
1520 }
1521 
cmd_handler_mi_wait_for_event(struct parser_exec_state * s)1522 static int cmd_handler_mi_wait_for_event(struct parser_exec_state *s)
1523 {
1524 	u32 cmd = cmd_val(s, 0);
1525 
1526 	if (!is_wait_for_flip_pending(cmd))
1527 		return 0;
1528 
1529 	patch_value(s, cmd_ptr(s, 0), MI_NOOP);
1530 	return 0;
1531 }
1532 
get_gma_bb_from_cmd(struct parser_exec_state * s,int index)1533 static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index)
1534 {
1535 	unsigned long addr;
1536 	unsigned long gma_high, gma_low;
1537 	struct intel_vgpu *vgpu = s->vgpu;
1538 	int gmadr_bytes = vgpu->gvt->device_info.gmadr_bytes_in_cmd;
1539 
1540 	if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) {
1541 		gvt_vgpu_err("invalid gma bytes %d\n", gmadr_bytes);
1542 		return INTEL_GVT_INVALID_ADDR;
1543 	}
1544 
1545 	gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK;
1546 	if (gmadr_bytes == 4) {
1547 		addr = gma_low;
1548 	} else {
1549 		gma_high = cmd_val(s, index + 1) & BATCH_BUFFER_ADDR_HIGH_MASK;
1550 		addr = (((unsigned long)gma_high) << 32) | gma_low;
1551 	}
1552 	return addr;
1553 }
1554 
cmd_address_audit(struct parser_exec_state * s,unsigned long guest_gma,int op_size,bool index_mode)1555 static inline int cmd_address_audit(struct parser_exec_state *s,
1556 		unsigned long guest_gma, int op_size, bool index_mode)
1557 {
1558 	struct intel_vgpu *vgpu = s->vgpu;
1559 	u32 max_surface_size = vgpu->gvt->device_info.max_surface_size;
1560 	int i;
1561 	int ret;
1562 
1563 	if (op_size > max_surface_size) {
1564 		gvt_vgpu_err("command address audit fail name %s\n",
1565 			s->info->name);
1566 		return -EFAULT;
1567 	}
1568 
1569 	if (index_mode)	{
1570 		if (guest_gma >= I915_GTT_PAGE_SIZE) {
1571 			ret = -EFAULT;
1572 			goto err;
1573 		}
1574 	} else if (!intel_gvt_ggtt_validate_range(vgpu, guest_gma, op_size)) {
1575 		ret = -EFAULT;
1576 		goto err;
1577 	}
1578 
1579 	return 0;
1580 
1581 err:
1582 	gvt_vgpu_err("cmd_parser: Malicious %s detected, addr=0x%lx, len=%d!\n",
1583 			s->info->name, guest_gma, op_size);
1584 
1585 	pr_err("cmd dump: ");
1586 	for (i = 0; i < cmd_length(s); i++) {
1587 		if (!(i % 4))
1588 			pr_err("\n%08x ", cmd_val(s, i));
1589 		else
1590 			pr_err("%08x ", cmd_val(s, i));
1591 	}
1592 	pr_err("\nvgpu%d: aperture 0x%llx - 0x%llx, hidden 0x%llx - 0x%llx\n",
1593 			vgpu->id,
1594 			vgpu_aperture_gmadr_base(vgpu),
1595 			vgpu_aperture_gmadr_end(vgpu),
1596 			vgpu_hidden_gmadr_base(vgpu),
1597 			vgpu_hidden_gmadr_end(vgpu));
1598 	return ret;
1599 }
1600 
cmd_handler_mi_store_data_imm(struct parser_exec_state * s)1601 static int cmd_handler_mi_store_data_imm(struct parser_exec_state *s)
1602 {
1603 	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
1604 	int op_size = (cmd_length(s) - 3) * sizeof(u32);
1605 	int core_id = (cmd_val(s, 2) & (1 << 0)) ? 1 : 0;
1606 	unsigned long gma, gma_low, gma_high;
1607 	u32 valid_len = CMD_LEN(2);
1608 	int ret = 0;
1609 
1610 	/* check ppggt */
1611 	if (!(cmd_val(s, 0) & (1 << 22)))
1612 		return 0;
1613 
1614 	/* check if QWORD */
1615 	if (DWORD_FIELD(0, 21, 21))
1616 		valid_len++;
1617 	ret = gvt_check_valid_cmd_length(cmd_length(s),
1618 			valid_len);
1619 	if (ret)
1620 		return ret;
1621 
1622 	gma = cmd_val(s, 2) & GENMASK(31, 2);
1623 
1624 	if (gmadr_bytes == 8) {
1625 		gma_low = cmd_val(s, 1) & GENMASK(31, 2);
1626 		gma_high = cmd_val(s, 2) & GENMASK(15, 0);
1627 		gma = (gma_high << 32) | gma_low;
1628 		core_id = (cmd_val(s, 1) & (1 << 0)) ? 1 : 0;
1629 	}
1630 	ret = cmd_address_audit(s, gma + op_size * core_id, op_size, false);
1631 	return ret;
1632 }
1633 
unexpected_cmd(struct parser_exec_state * s)1634 static inline int unexpected_cmd(struct parser_exec_state *s)
1635 {
1636 	struct intel_vgpu *vgpu = s->vgpu;
1637 
1638 	gvt_vgpu_err("Unexpected %s in command buffer!\n", s->info->name);
1639 
1640 	return -EBADRQC;
1641 }
1642 
cmd_handler_mi_semaphore_wait(struct parser_exec_state * s)1643 static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s)
1644 {
1645 	return unexpected_cmd(s);
1646 }
1647 
cmd_handler_mi_report_perf_count(struct parser_exec_state * s)1648 static int cmd_handler_mi_report_perf_count(struct parser_exec_state *s)
1649 {
1650 	return unexpected_cmd(s);
1651 }
1652 
cmd_handler_mi_op_2e(struct parser_exec_state * s)1653 static int cmd_handler_mi_op_2e(struct parser_exec_state *s)
1654 {
1655 	return unexpected_cmd(s);
1656 }
1657 
cmd_handler_mi_op_2f(struct parser_exec_state * s)1658 static int cmd_handler_mi_op_2f(struct parser_exec_state *s)
1659 {
1660 	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
1661 	int op_size = (1 << ((cmd_val(s, 0) & GENMASK(20, 19)) >> 19)) *
1662 			sizeof(u32);
1663 	unsigned long gma, gma_high;
1664 	u32 valid_len = CMD_LEN(1);
1665 	int ret = 0;
1666 
1667 	if (!(cmd_val(s, 0) & (1 << 22)))
1668 		return ret;
1669 
1670 	/* check inline data */
1671 	if (cmd_val(s, 0) & BIT(18))
1672 		valid_len = CMD_LEN(9);
1673 	ret = gvt_check_valid_cmd_length(cmd_length(s),
1674 			valid_len);
1675 	if (ret)
1676 		return ret;
1677 
1678 	gma = cmd_val(s, 1) & GENMASK(31, 2);
1679 	if (gmadr_bytes == 8) {
1680 		gma_high = cmd_val(s, 2) & GENMASK(15, 0);
1681 		gma = (gma_high << 32) | gma;
1682 	}
1683 	ret = cmd_address_audit(s, gma, op_size, false);
1684 	return ret;
1685 }
1686 
cmd_handler_mi_store_data_index(struct parser_exec_state * s)1687 static int cmd_handler_mi_store_data_index(struct parser_exec_state *s)
1688 {
1689 	return unexpected_cmd(s);
1690 }
1691 
cmd_handler_mi_clflush(struct parser_exec_state * s)1692 static int cmd_handler_mi_clflush(struct parser_exec_state *s)
1693 {
1694 	return unexpected_cmd(s);
1695 }
1696 
cmd_handler_mi_conditional_batch_buffer_end(struct parser_exec_state * s)1697 static int cmd_handler_mi_conditional_batch_buffer_end(
1698 		struct parser_exec_state *s)
1699 {
1700 	return unexpected_cmd(s);
1701 }
1702 
cmd_handler_mi_update_gtt(struct parser_exec_state * s)1703 static int cmd_handler_mi_update_gtt(struct parser_exec_state *s)
1704 {
1705 	return unexpected_cmd(s);
1706 }
1707 
cmd_handler_mi_flush_dw(struct parser_exec_state * s)1708 static int cmd_handler_mi_flush_dw(struct parser_exec_state *s)
1709 {
1710 	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
1711 	unsigned long gma;
1712 	bool index_mode = false;
1713 	int ret = 0;
1714 	u32 hws_pga, val;
1715 	u32 valid_len = CMD_LEN(2);
1716 
1717 	ret = gvt_check_valid_cmd_length(cmd_length(s),
1718 			valid_len);
1719 	if (ret) {
1720 		/* Check again for Qword */
1721 		ret = gvt_check_valid_cmd_length(cmd_length(s),
1722 			++valid_len);
1723 		return ret;
1724 	}
1725 
1726 	/* Check post-sync and ppgtt bit */
1727 	if (((cmd_val(s, 0) >> 14) & 0x3) && (cmd_val(s, 1) & (1 << 2))) {
1728 		gma = cmd_val(s, 1) & GENMASK(31, 3);
1729 		if (gmadr_bytes == 8)
1730 			gma |= (cmd_val(s, 2) & GENMASK(15, 0)) << 32;
1731 		/* Store Data Index */
1732 		if (cmd_val(s, 0) & (1 << 21))
1733 			index_mode = true;
1734 		ret = cmd_address_audit(s, gma, sizeof(u64), index_mode);
1735 		if (ret)
1736 			return ret;
1737 		if (index_mode) {
1738 			hws_pga = s->vgpu->hws_pga[s->engine->id];
1739 			gma = hws_pga + gma;
1740 			patch_value(s, cmd_ptr(s, 1), gma);
1741 			val = cmd_val(s, 0) & (~(1 << 21));
1742 			patch_value(s, cmd_ptr(s, 0), val);
1743 		}
1744 	}
1745 	/* Check notify bit */
1746 	if ((cmd_val(s, 0) & (1 << 8)))
1747 		set_bit(cmd_interrupt_events[s->engine->id].mi_flush_dw,
1748 			s->workload->pending_events);
1749 	return ret;
1750 }
1751 
addr_type_update_snb(struct parser_exec_state * s)1752 static void addr_type_update_snb(struct parser_exec_state *s)
1753 {
1754 	if ((s->buf_type == RING_BUFFER_INSTRUCTION) &&
1755 			(BATCH_BUFFER_ADR_SPACE_BIT(cmd_val(s, 0)) == 1)) {
1756 		s->buf_addr_type = PPGTT_BUFFER;
1757 	}
1758 }
1759 
1760 
copy_gma_to_hva(struct intel_vgpu * vgpu,struct intel_vgpu_mm * mm,unsigned long gma,unsigned long end_gma,void * va)1761 static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm,
1762 		unsigned long gma, unsigned long end_gma, void *va)
1763 {
1764 	unsigned long copy_len, offset;
1765 	unsigned long len = 0;
1766 	unsigned long gpa;
1767 
1768 	while (gma != end_gma) {
1769 		gpa = intel_vgpu_gma_to_gpa(mm, gma);
1770 		if (gpa == INTEL_GVT_INVALID_ADDR) {
1771 			gvt_vgpu_err("invalid gma address: %lx\n", gma);
1772 			return -EFAULT;
1773 		}
1774 
1775 		offset = gma & (I915_GTT_PAGE_SIZE - 1);
1776 
1777 		copy_len = (end_gma - gma) >= (I915_GTT_PAGE_SIZE - offset) ?
1778 			I915_GTT_PAGE_SIZE - offset : end_gma - gma;
1779 
1780 		intel_gvt_read_gpa(vgpu, gpa, va + len, copy_len);
1781 
1782 		len += copy_len;
1783 		gma += copy_len;
1784 	}
1785 	return len;
1786 }
1787 
1788 
1789 /*
1790  * Check whether a batch buffer needs to be scanned. Currently
1791  * the only criteria is based on privilege.
1792  */
batch_buffer_needs_scan(struct parser_exec_state * s)1793 static int batch_buffer_needs_scan(struct parser_exec_state *s)
1794 {
1795 	/* Decide privilege based on address space */
1796 	if (cmd_val(s, 0) & BIT(8) &&
1797 	    !(s->vgpu->scan_nonprivbb & s->engine->mask))
1798 		return 0;
1799 
1800 	return 1;
1801 }
1802 
repr_addr_type(unsigned int type)1803 static const char *repr_addr_type(unsigned int type)
1804 {
1805 	return type == PPGTT_BUFFER ? "ppgtt" : "ggtt";
1806 }
1807 
find_bb_size(struct parser_exec_state * s,unsigned long * bb_size,unsigned long * bb_end_cmd_offset)1808 static int find_bb_size(struct parser_exec_state *s,
1809 			unsigned long *bb_size,
1810 			unsigned long *bb_end_cmd_offset)
1811 {
1812 	unsigned long gma = 0;
1813 	const struct cmd_info *info;
1814 	u32 cmd_len = 0;
1815 	bool bb_end = false;
1816 	struct intel_vgpu *vgpu = s->vgpu;
1817 	u32 cmd;
1818 	struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ?
1819 		s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm;
1820 
1821 	*bb_size = 0;
1822 	*bb_end_cmd_offset = 0;
1823 
1824 	/* get the start gm address of the batch buffer */
1825 	gma = get_gma_bb_from_cmd(s, 1);
1826 	if (gma == INTEL_GVT_INVALID_ADDR)
1827 		return -EFAULT;
1828 
1829 	cmd = cmd_val(s, 0);
1830 	info = get_cmd_info(s->vgpu->gvt, cmd, s->engine);
1831 	if (info == NULL) {
1832 		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %s, workload=%p\n",
1833 			     cmd, get_opcode(cmd, s->engine),
1834 			     repr_addr_type(s->buf_addr_type),
1835 			     s->engine->name, s->workload);
1836 		return -EBADRQC;
1837 	}
1838 	do {
1839 		if (copy_gma_to_hva(s->vgpu, mm,
1840 				    gma, gma + 4, &cmd) < 0)
1841 			return -EFAULT;
1842 		info = get_cmd_info(s->vgpu->gvt, cmd, s->engine);
1843 		if (info == NULL) {
1844 			gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %s, workload=%p\n",
1845 				     cmd, get_opcode(cmd, s->engine),
1846 				     repr_addr_type(s->buf_addr_type),
1847 				     s->engine->name, s->workload);
1848 			return -EBADRQC;
1849 		}
1850 
1851 		if (info->opcode == OP_MI_BATCH_BUFFER_END) {
1852 			bb_end = true;
1853 		} else if (info->opcode == OP_MI_BATCH_BUFFER_START) {
1854 			if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0)
1855 				/* chained batch buffer */
1856 				bb_end = true;
1857 		}
1858 
1859 		if (bb_end)
1860 			*bb_end_cmd_offset = *bb_size;
1861 
1862 		cmd_len = get_cmd_length(info, cmd) << 2;
1863 		*bb_size += cmd_len;
1864 		gma += cmd_len;
1865 	} while (!bb_end);
1866 
1867 	return 0;
1868 }
1869 
audit_bb_end(struct parser_exec_state * s,void * va)1870 static int audit_bb_end(struct parser_exec_state *s, void *va)
1871 {
1872 	struct intel_vgpu *vgpu = s->vgpu;
1873 	u32 cmd = *(u32 *)va;
1874 	const struct cmd_info *info;
1875 
1876 	info = get_cmd_info(s->vgpu->gvt, cmd, s->engine);
1877 	if (info == NULL) {
1878 		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %s, workload=%p\n",
1879 			     cmd, get_opcode(cmd, s->engine),
1880 			     repr_addr_type(s->buf_addr_type),
1881 			     s->engine->name, s->workload);
1882 		return -EBADRQC;
1883 	}
1884 
1885 	if ((info->opcode == OP_MI_BATCH_BUFFER_END) ||
1886 	    ((info->opcode == OP_MI_BATCH_BUFFER_START) &&
1887 	     (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0)))
1888 		return 0;
1889 
1890 	return -EBADRQC;
1891 }
1892 
perform_bb_shadow(struct parser_exec_state * s)1893 static int perform_bb_shadow(struct parser_exec_state *s)
1894 {
1895 	struct intel_vgpu *vgpu = s->vgpu;
1896 	struct intel_vgpu_shadow_bb *bb;
1897 	unsigned long gma = 0;
1898 	unsigned long bb_size;
1899 	unsigned long bb_end_cmd_offset;
1900 	int ret = 0;
1901 	struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ?
1902 		s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm;
1903 	unsigned long start_offset = 0;
1904 
1905 	/* get the start gm address of the batch buffer */
1906 	gma = get_gma_bb_from_cmd(s, 1);
1907 	if (gma == INTEL_GVT_INVALID_ADDR)
1908 		return -EFAULT;
1909 
1910 	ret = find_bb_size(s, &bb_size, &bb_end_cmd_offset);
1911 	if (ret)
1912 		return ret;
1913 
1914 	bb = kzalloc(sizeof(*bb), GFP_KERNEL);
1915 	if (!bb)
1916 		return -ENOMEM;
1917 
1918 	bb->ppgtt = (s->buf_addr_type == GTT_BUFFER) ? false : true;
1919 
1920 	/* the start_offset stores the batch buffer's start gma's
1921 	 * offset relative to page boundary. so for non-privileged batch
1922 	 * buffer, the shadowed gem object holds exactly the same page
1923 	 * layout as original gem object. This is for the convience of
1924 	 * replacing the whole non-privilged batch buffer page to this
1925 	 * shadowed one in PPGTT at the same gma address. (this replacing
1926 	 * action is not implemented yet now, but may be necessary in
1927 	 * future).
1928 	 * for prileged batch buffer, we just change start gma address to
1929 	 * that of shadowed page.
1930 	 */
1931 	if (bb->ppgtt)
1932 		start_offset = gma & ~I915_GTT_PAGE_MASK;
1933 
1934 	bb->obj = i915_gem_object_create_shmem(s->engine->i915,
1935 					       round_up(bb_size + start_offset,
1936 							PAGE_SIZE));
1937 	if (IS_ERR(bb->obj)) {
1938 		ret = PTR_ERR(bb->obj);
1939 		goto err_free_bb;
1940 	}
1941 
1942 	bb->va = i915_gem_object_pin_map(bb->obj, I915_MAP_WB);
1943 	if (IS_ERR(bb->va)) {
1944 		ret = PTR_ERR(bb->va);
1945 		goto err_free_obj;
1946 	}
1947 
1948 	ret = copy_gma_to_hva(s->vgpu, mm,
1949 			      gma, gma + bb_size,
1950 			      bb->va + start_offset);
1951 	if (ret < 0) {
1952 		gvt_vgpu_err("fail to copy guest ring buffer\n");
1953 		ret = -EFAULT;
1954 		goto err_unmap;
1955 	}
1956 
1957 	ret = audit_bb_end(s, bb->va + start_offset + bb_end_cmd_offset);
1958 	if (ret)
1959 		goto err_unmap;
1960 
1961 	i915_gem_object_unlock(bb->obj);
1962 	INIT_LIST_HEAD(&bb->list);
1963 	list_add(&bb->list, &s->workload->shadow_bb);
1964 
1965 	bb->bb_start_cmd_va = s->ip_va;
1966 
1967 	if ((s->buf_type == BATCH_BUFFER_INSTRUCTION) && (!s->is_ctx_wa))
1968 		bb->bb_offset = s->ip_va - s->rb_va;
1969 	else
1970 		bb->bb_offset = 0;
1971 
1972 	/*
1973 	 * ip_va saves the virtual address of the shadow batch buffer, while
1974 	 * ip_gma saves the graphics address of the original batch buffer.
1975 	 * As the shadow batch buffer is just a copy from the originial one,
1976 	 * it should be right to use shadow batch buffer'va and original batch
1977 	 * buffer's gma in pair. After all, we don't want to pin the shadow
1978 	 * buffer here (too early).
1979 	 */
1980 	s->ip_va = bb->va + start_offset;
1981 	s->ip_gma = gma;
1982 	return 0;
1983 err_unmap:
1984 	i915_gem_object_unpin_map(bb->obj);
1985 err_free_obj:
1986 	i915_gem_object_put(bb->obj);
1987 err_free_bb:
1988 	kfree(bb);
1989 	return ret;
1990 }
1991 
cmd_handler_mi_batch_buffer_start(struct parser_exec_state * s)1992 static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s)
1993 {
1994 	bool second_level;
1995 	int ret = 0;
1996 	struct intel_vgpu *vgpu = s->vgpu;
1997 
1998 	if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
1999 		gvt_vgpu_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n");
2000 		return -EFAULT;
2001 	}
2002 
2003 	second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1;
2004 	if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) {
2005 		gvt_vgpu_err("Jumping to 2nd level BB from RB is not allowed\n");
2006 		return -EFAULT;
2007 	}
2008 
2009 	s->saved_buf_addr_type = s->buf_addr_type;
2010 	addr_type_update_snb(s);
2011 	if (s->buf_type == RING_BUFFER_INSTRUCTION) {
2012 		s->ret_ip_gma_ring = s->ip_gma + cmd_length(s) * sizeof(u32);
2013 		s->buf_type = BATCH_BUFFER_INSTRUCTION;
2014 	} else if (second_level) {
2015 		s->buf_type = BATCH_BUFFER_2ND_LEVEL;
2016 		s->ret_ip_gma_bb = s->ip_gma + cmd_length(s) * sizeof(u32);
2017 		s->ret_bb_va = s->ip_va + cmd_length(s) * sizeof(u32);
2018 	}
2019 
2020 	if (batch_buffer_needs_scan(s)) {
2021 		ret = perform_bb_shadow(s);
2022 		if (ret < 0)
2023 			gvt_vgpu_err("invalid shadow batch buffer\n");
2024 	} else {
2025 		/* emulate a batch buffer end to do return right */
2026 		ret = cmd_handler_mi_batch_buffer_end(s);
2027 		if (ret < 0)
2028 			return ret;
2029 	}
2030 	return ret;
2031 }
2032 
2033 static int mi_noop_index;
2034 
2035 static const struct cmd_info cmd_info[] = {
2036 	{"MI_NOOP", OP_MI_NOOP, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
2037 
2038 	{"MI_SET_PREDICATE", OP_MI_SET_PREDICATE, F_LEN_CONST, R_ALL, D_ALL,
2039 		0, 1, NULL},
2040 
2041 	{"MI_USER_INTERRUPT", OP_MI_USER_INTERRUPT, F_LEN_CONST, R_ALL, D_ALL,
2042 		0, 1, cmd_handler_mi_user_interrupt},
2043 
2044 	{"MI_WAIT_FOR_EVENT", OP_MI_WAIT_FOR_EVENT, F_LEN_CONST, R_RCS | R_BCS,
2045 		D_ALL, 0, 1, cmd_handler_mi_wait_for_event},
2046 
2047 	{"MI_FLUSH", OP_MI_FLUSH, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
2048 
2049 	{"MI_ARB_CHECK", OP_MI_ARB_CHECK, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
2050 		NULL},
2051 
2052 	{"MI_RS_CONTROL", OP_MI_RS_CONTROL, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
2053 		NULL},
2054 
2055 	{"MI_REPORT_HEAD", OP_MI_REPORT_HEAD, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
2056 		NULL},
2057 
2058 	{"MI_ARB_ON_OFF", OP_MI_ARB_ON_OFF, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
2059 		NULL},
2060 
2061 	{"MI_URB_ATOMIC_ALLOC", OP_MI_URB_ATOMIC_ALLOC, F_LEN_CONST, R_RCS,
2062 		D_ALL, 0, 1, NULL},
2063 
2064 	{"MI_BATCH_BUFFER_END", OP_MI_BATCH_BUFFER_END,
2065 		F_IP_ADVANCE_CUSTOM | F_LEN_CONST, R_ALL, D_ALL, 0, 1,
2066 		cmd_handler_mi_batch_buffer_end},
2067 
2068 	{"MI_SUSPEND_FLUSH", OP_MI_SUSPEND_FLUSH, F_LEN_CONST, R_ALL, D_ALL,
2069 		0, 1, NULL},
2070 
2071 	{"MI_PREDICATE", OP_MI_PREDICATE, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
2072 		NULL},
2073 
2074 	{"MI_TOPOLOGY_FILTER", OP_MI_TOPOLOGY_FILTER, F_LEN_CONST, R_ALL,
2075 		D_ALL, 0, 1, NULL},
2076 
2077 	{"MI_SET_APPID", OP_MI_SET_APPID, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
2078 		NULL},
2079 
2080 	{"MI_RS_CONTEXT", OP_MI_RS_CONTEXT, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
2081 		NULL},
2082 
2083 	{"MI_DISPLAY_FLIP", OP_MI_DISPLAY_FLIP, F_LEN_VAR,
2084 		R_RCS | R_BCS, D_ALL, 0, 8, cmd_handler_mi_display_flip},
2085 
2086 	{"MI_SEMAPHORE_MBOX", OP_MI_SEMAPHORE_MBOX, F_LEN_VAR | F_LEN_VAR_FIXED,
2087 		R_ALL, D_ALL, 0, 8, NULL, CMD_LEN(1)},
2088 
2089 	{"MI_MATH", OP_MI_MATH, F_LEN_VAR, R_ALL, D_ALL, 0, 8, NULL},
2090 
2091 	{"MI_URB_CLEAR", OP_MI_URB_CLEAR, F_LEN_VAR | F_LEN_VAR_FIXED, R_RCS,
2092 		D_ALL, 0, 8, NULL, CMD_LEN(0)},
2093 
2094 	{"MI_SEMAPHORE_SIGNAL", OP_MI_SEMAPHORE_SIGNAL,
2095 		F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS, 0, 8,
2096 		NULL, CMD_LEN(0)},
2097 
2098 	{"MI_SEMAPHORE_WAIT", OP_MI_SEMAPHORE_WAIT,
2099 		F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS, ADDR_FIX_1(2),
2100 		8, cmd_handler_mi_semaphore_wait, CMD_LEN(2)},
2101 
2102 	{"MI_STORE_DATA_IMM", OP_MI_STORE_DATA_IMM, F_LEN_VAR, R_ALL, D_BDW_PLUS,
2103 		ADDR_FIX_1(1), 10, cmd_handler_mi_store_data_imm},
2104 
2105 	{"MI_STORE_DATA_INDEX", OP_MI_STORE_DATA_INDEX, F_LEN_VAR, R_ALL, D_ALL,
2106 		0, 8, cmd_handler_mi_store_data_index},
2107 
2108 	{"MI_LOAD_REGISTER_IMM", OP_MI_LOAD_REGISTER_IMM, F_LEN_VAR, R_ALL,
2109 		D_ALL, 0, 8, cmd_handler_lri},
2110 
2111 	{"MI_UPDATE_GTT", OP_MI_UPDATE_GTT, F_LEN_VAR, R_ALL, D_BDW_PLUS, 0, 10,
2112 		cmd_handler_mi_update_gtt},
2113 
2114 	{"MI_STORE_REGISTER_MEM", OP_MI_STORE_REGISTER_MEM,
2115 		F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8,
2116 		cmd_handler_srm, CMD_LEN(2)},
2117 
2118 	{"MI_FLUSH_DW", OP_MI_FLUSH_DW, F_LEN_VAR, R_ALL, D_ALL, 0, 6,
2119 		cmd_handler_mi_flush_dw},
2120 
2121 	{"MI_CLFLUSH", OP_MI_CLFLUSH, F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(1),
2122 		10, cmd_handler_mi_clflush},
2123 
2124 	{"MI_REPORT_PERF_COUNT", OP_MI_REPORT_PERF_COUNT,
2125 		F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(1), 6,
2126 		cmd_handler_mi_report_perf_count, CMD_LEN(2)},
2127 
2128 	{"MI_LOAD_REGISTER_MEM", OP_MI_LOAD_REGISTER_MEM,
2129 		F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8,
2130 		cmd_handler_lrm, CMD_LEN(2)},
2131 
2132 	{"MI_LOAD_REGISTER_REG", OP_MI_LOAD_REGISTER_REG,
2133 		F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, 0, 8,
2134 		cmd_handler_lrr, CMD_LEN(1)},
2135 
2136 	{"MI_RS_STORE_DATA_IMM", OP_MI_RS_STORE_DATA_IMM,
2137 		F_LEN_VAR | F_LEN_VAR_FIXED, R_RCS, D_ALL, 0,
2138 		8, NULL, CMD_LEN(2)},
2139 
2140 	{"MI_LOAD_URB_MEM", OP_MI_LOAD_URB_MEM, F_LEN_VAR | F_LEN_VAR_FIXED,
2141 		R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL, CMD_LEN(2)},
2142 
2143 	{"MI_STORE_URM_MEM", OP_MI_STORE_URM_MEM, F_LEN_VAR, R_RCS, D_ALL,
2144 		ADDR_FIX_1(2), 8, NULL},
2145 
2146 	{"MI_OP_2E", OP_MI_2E, F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS,
2147 		ADDR_FIX_2(1, 2), 8, cmd_handler_mi_op_2e, CMD_LEN(3)},
2148 
2149 	{"MI_OP_2F", OP_MI_2F, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_1(1),
2150 		8, cmd_handler_mi_op_2f},
2151 
2152 	{"MI_BATCH_BUFFER_START", OP_MI_BATCH_BUFFER_START,
2153 		F_IP_ADVANCE_CUSTOM, R_ALL, D_ALL, 0, 8,
2154 		cmd_handler_mi_batch_buffer_start},
2155 
2156 	{"MI_CONDITIONAL_BATCH_BUFFER_END", OP_MI_CONDITIONAL_BATCH_BUFFER_END,
2157 		F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8,
2158 		cmd_handler_mi_conditional_batch_buffer_end, CMD_LEN(2)},
2159 
2160 	{"MI_LOAD_SCAN_LINES_INCL", OP_MI_LOAD_SCAN_LINES_INCL, F_LEN_CONST,
2161 		R_RCS | R_BCS, D_ALL, 0, 2, NULL},
2162 
2163 	{"XY_SETUP_BLT", OP_XY_SETUP_BLT, F_LEN_VAR, R_BCS, D_ALL,
2164 		ADDR_FIX_2(4, 7), 8, NULL},
2165 
2166 	{"XY_SETUP_CLIP_BLT", OP_XY_SETUP_CLIP_BLT, F_LEN_VAR, R_BCS, D_ALL,
2167 		0, 8, NULL},
2168 
2169 	{"XY_SETUP_MONO_PATTERN_SL_BLT", OP_XY_SETUP_MONO_PATTERN_SL_BLT,
2170 		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
2171 
2172 	{"XY_PIXEL_BLT", OP_XY_PIXEL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL},
2173 
2174 	{"XY_SCANLINES_BLT", OP_XY_SCANLINES_BLT, F_LEN_VAR, R_BCS, D_ALL,
2175 		0, 8, NULL},
2176 
2177 	{"XY_TEXT_BLT", OP_XY_TEXT_BLT, F_LEN_VAR, R_BCS, D_ALL,
2178 		ADDR_FIX_1(3), 8, NULL},
2179 
2180 	{"XY_TEXT_IMMEDIATE_BLT", OP_XY_TEXT_IMMEDIATE_BLT, F_LEN_VAR, R_BCS,
2181 		D_ALL, 0, 8, NULL},
2182 
2183 	{"XY_COLOR_BLT", OP_XY_COLOR_BLT, F_LEN_VAR, R_BCS, D_ALL,
2184 		ADDR_FIX_1(4), 8, NULL},
2185 
2186 	{"XY_PAT_BLT", OP_XY_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL,
2187 		ADDR_FIX_2(4, 5), 8, NULL},
2188 
2189 	{"XY_MONO_PAT_BLT", OP_XY_MONO_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL,
2190 		ADDR_FIX_1(4), 8, NULL},
2191 
2192 	{"XY_SRC_COPY_BLT", OP_XY_SRC_COPY_BLT, F_LEN_VAR, R_BCS, D_ALL,
2193 		ADDR_FIX_2(4, 7), 8, NULL},
2194 
2195 	{"XY_MONO_SRC_COPY_BLT", OP_XY_MONO_SRC_COPY_BLT, F_LEN_VAR, R_BCS,
2196 		D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
2197 
2198 	{"XY_FULL_BLT", OP_XY_FULL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL},
2199 
2200 	{"XY_FULL_MONO_SRC_BLT", OP_XY_FULL_MONO_SRC_BLT, F_LEN_VAR, R_BCS,
2201 		D_ALL, ADDR_FIX_3(4, 5, 8), 8, NULL},
2202 
2203 	{"XY_FULL_MONO_PATTERN_BLT", OP_XY_FULL_MONO_PATTERN_BLT, F_LEN_VAR,
2204 		R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
2205 
2206 	{"XY_FULL_MONO_PATTERN_MONO_SRC_BLT",
2207 		OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT,
2208 		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
2209 
2210 	{"XY_MONO_PAT_FIXED_BLT", OP_XY_MONO_PAT_FIXED_BLT, F_LEN_VAR, R_BCS,
2211 		D_ALL, ADDR_FIX_1(4), 8, NULL},
2212 
2213 	{"XY_MONO_SRC_COPY_IMMEDIATE_BLT", OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT,
2214 		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
2215 
2216 	{"XY_PAT_BLT_IMMEDIATE", OP_XY_PAT_BLT_IMMEDIATE, F_LEN_VAR, R_BCS,
2217 		D_ALL, ADDR_FIX_1(4), 8, NULL},
2218 
2219 	{"XY_SRC_COPY_CHROMA_BLT", OP_XY_SRC_COPY_CHROMA_BLT, F_LEN_VAR, R_BCS,
2220 		D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
2221 
2222 	{"XY_FULL_IMMEDIATE_PATTERN_BLT", OP_XY_FULL_IMMEDIATE_PATTERN_BLT,
2223 		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
2224 
2225 	{"XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT",
2226 		OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT,
2227 		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
2228 
2229 	{"XY_PAT_CHROMA_BLT", OP_XY_PAT_CHROMA_BLT, F_LEN_VAR, R_BCS, D_ALL,
2230 		ADDR_FIX_2(4, 5), 8, NULL},
2231 
2232 	{"XY_PAT_CHROMA_BLT_IMMEDIATE", OP_XY_PAT_CHROMA_BLT_IMMEDIATE,
2233 		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
2234 
2235 	{"3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP",
2236 		OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP,
2237 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2238 
2239 	{"3DSTATE_VIEWPORT_STATE_POINTERS_CC",
2240 		OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC,
2241 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2242 
2243 	{"3DSTATE_BLEND_STATE_POINTERS",
2244 		OP_3DSTATE_BLEND_STATE_POINTERS,
2245 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2246 
2247 	{"3DSTATE_DEPTH_STENCIL_STATE_POINTERS",
2248 		OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS,
2249 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2250 
2251 	{"3DSTATE_BINDING_TABLE_POINTERS_VS",
2252 		OP_3DSTATE_BINDING_TABLE_POINTERS_VS,
2253 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2254 
2255 	{"3DSTATE_BINDING_TABLE_POINTERS_HS",
2256 		OP_3DSTATE_BINDING_TABLE_POINTERS_HS,
2257 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2258 
2259 	{"3DSTATE_BINDING_TABLE_POINTERS_DS",
2260 		OP_3DSTATE_BINDING_TABLE_POINTERS_DS,
2261 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2262 
2263 	{"3DSTATE_BINDING_TABLE_POINTERS_GS",
2264 		OP_3DSTATE_BINDING_TABLE_POINTERS_GS,
2265 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2266 
2267 	{"3DSTATE_BINDING_TABLE_POINTERS_PS",
2268 		OP_3DSTATE_BINDING_TABLE_POINTERS_PS,
2269 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2270 
2271 	{"3DSTATE_SAMPLER_STATE_POINTERS_VS",
2272 		OP_3DSTATE_SAMPLER_STATE_POINTERS_VS,
2273 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2274 
2275 	{"3DSTATE_SAMPLER_STATE_POINTERS_HS",
2276 		OP_3DSTATE_SAMPLER_STATE_POINTERS_HS,
2277 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2278 
2279 	{"3DSTATE_SAMPLER_STATE_POINTERS_DS",
2280 		OP_3DSTATE_SAMPLER_STATE_POINTERS_DS,
2281 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2282 
2283 	{"3DSTATE_SAMPLER_STATE_POINTERS_GS",
2284 		OP_3DSTATE_SAMPLER_STATE_POINTERS_GS,
2285 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2286 
2287 	{"3DSTATE_SAMPLER_STATE_POINTERS_PS",
2288 		OP_3DSTATE_SAMPLER_STATE_POINTERS_PS,
2289 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2290 
2291 	{"3DSTATE_URB_VS", OP_3DSTATE_URB_VS, F_LEN_VAR, R_RCS, D_ALL,
2292 		0, 8, NULL},
2293 
2294 	{"3DSTATE_URB_HS", OP_3DSTATE_URB_HS, F_LEN_VAR, R_RCS, D_ALL,
2295 		0, 8, NULL},
2296 
2297 	{"3DSTATE_URB_DS", OP_3DSTATE_URB_DS, F_LEN_VAR, R_RCS, D_ALL,
2298 		0, 8, NULL},
2299 
2300 	{"3DSTATE_URB_GS", OP_3DSTATE_URB_GS, F_LEN_VAR, R_RCS, D_ALL,
2301 		0, 8, NULL},
2302 
2303 	{"3DSTATE_GATHER_CONSTANT_VS", OP_3DSTATE_GATHER_CONSTANT_VS,
2304 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2305 
2306 	{"3DSTATE_GATHER_CONSTANT_GS", OP_3DSTATE_GATHER_CONSTANT_GS,
2307 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2308 
2309 	{"3DSTATE_GATHER_CONSTANT_HS", OP_3DSTATE_GATHER_CONSTANT_HS,
2310 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2311 
2312 	{"3DSTATE_GATHER_CONSTANT_DS", OP_3DSTATE_GATHER_CONSTANT_DS,
2313 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2314 
2315 	{"3DSTATE_GATHER_CONSTANT_PS", OP_3DSTATE_GATHER_CONSTANT_PS,
2316 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2317 
2318 	{"3DSTATE_DX9_CONSTANTF_VS", OP_3DSTATE_DX9_CONSTANTF_VS,
2319 		F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL},
2320 
2321 	{"3DSTATE_DX9_CONSTANTF_PS", OP_3DSTATE_DX9_CONSTANTF_PS,
2322 		F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL},
2323 
2324 	{"3DSTATE_DX9_CONSTANTI_VS", OP_3DSTATE_DX9_CONSTANTI_VS,
2325 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2326 
2327 	{"3DSTATE_DX9_CONSTANTI_PS", OP_3DSTATE_DX9_CONSTANTI_PS,
2328 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2329 
2330 	{"3DSTATE_DX9_CONSTANTB_VS", OP_3DSTATE_DX9_CONSTANTB_VS,
2331 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2332 
2333 	{"3DSTATE_DX9_CONSTANTB_PS", OP_3DSTATE_DX9_CONSTANTB_PS,
2334 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2335 
2336 	{"3DSTATE_DX9_LOCAL_VALID_VS", OP_3DSTATE_DX9_LOCAL_VALID_VS,
2337 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2338 
2339 	{"3DSTATE_DX9_LOCAL_VALID_PS", OP_3DSTATE_DX9_LOCAL_VALID_PS,
2340 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2341 
2342 	{"3DSTATE_DX9_GENERATE_ACTIVE_VS", OP_3DSTATE_DX9_GENERATE_ACTIVE_VS,
2343 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2344 
2345 	{"3DSTATE_DX9_GENERATE_ACTIVE_PS", OP_3DSTATE_DX9_GENERATE_ACTIVE_PS,
2346 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2347 
2348 	{"3DSTATE_BINDING_TABLE_EDIT_VS", OP_3DSTATE_BINDING_TABLE_EDIT_VS,
2349 		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
2350 
2351 	{"3DSTATE_BINDING_TABLE_EDIT_GS", OP_3DSTATE_BINDING_TABLE_EDIT_GS,
2352 		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
2353 
2354 	{"3DSTATE_BINDING_TABLE_EDIT_HS", OP_3DSTATE_BINDING_TABLE_EDIT_HS,
2355 		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
2356 
2357 	{"3DSTATE_BINDING_TABLE_EDIT_DS", OP_3DSTATE_BINDING_TABLE_EDIT_DS,
2358 		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
2359 
2360 	{"3DSTATE_BINDING_TABLE_EDIT_PS", OP_3DSTATE_BINDING_TABLE_EDIT_PS,
2361 		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
2362 
2363 	{"3DSTATE_VF_INSTANCING", OP_3DSTATE_VF_INSTANCING, F_LEN_VAR, R_RCS,
2364 		D_BDW_PLUS, 0, 8, NULL},
2365 
2366 	{"3DSTATE_VF_SGVS", OP_3DSTATE_VF_SGVS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
2367 		NULL},
2368 
2369 	{"3DSTATE_VF_TOPOLOGY", OP_3DSTATE_VF_TOPOLOGY, F_LEN_VAR, R_RCS,
2370 		D_BDW_PLUS, 0, 8, NULL},
2371 
2372 	{"3DSTATE_WM_CHROMAKEY", OP_3DSTATE_WM_CHROMAKEY, F_LEN_VAR, R_RCS,
2373 		D_BDW_PLUS, 0, 8, NULL},
2374 
2375 	{"3DSTATE_PS_BLEND", OP_3DSTATE_PS_BLEND, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0,
2376 		8, NULL},
2377 
2378 	{"3DSTATE_WM_DEPTH_STENCIL", OP_3DSTATE_WM_DEPTH_STENCIL, F_LEN_VAR,
2379 		R_RCS, D_BDW_PLUS, 0, 8, NULL},
2380 
2381 	{"3DSTATE_PS_EXTRA", OP_3DSTATE_PS_EXTRA, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0,
2382 		8, NULL},
2383 
2384 	{"3DSTATE_RASTER", OP_3DSTATE_RASTER, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
2385 		NULL},
2386 
2387 	{"3DSTATE_SBE_SWIZ", OP_3DSTATE_SBE_SWIZ, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
2388 		NULL},
2389 
2390 	{"3DSTATE_WM_HZ_OP", OP_3DSTATE_WM_HZ_OP, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
2391 		NULL},
2392 
2393 	{"3DSTATE_VERTEX_BUFFERS", OP_3DSTATE_VERTEX_BUFFERS, F_LEN_VAR, R_RCS,
2394 		D_BDW_PLUS, 0, 8, NULL},
2395 
2396 	{"3DSTATE_VERTEX_ELEMENTS", OP_3DSTATE_VERTEX_ELEMENTS, F_LEN_VAR,
2397 		R_RCS, D_ALL, 0, 8, NULL},
2398 
2399 	{"3DSTATE_INDEX_BUFFER", OP_3DSTATE_INDEX_BUFFER, F_LEN_VAR, R_RCS,
2400 		D_BDW_PLUS, ADDR_FIX_1(2), 8, NULL},
2401 
2402 	{"3DSTATE_VF_STATISTICS", OP_3DSTATE_VF_STATISTICS, F_LEN_CONST,
2403 		R_RCS, D_ALL, 0, 1, NULL},
2404 
2405 	{"3DSTATE_VF", OP_3DSTATE_VF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2406 
2407 	{"3DSTATE_CC_STATE_POINTERS", OP_3DSTATE_CC_STATE_POINTERS, F_LEN_VAR,
2408 		R_RCS, D_ALL, 0, 8, NULL},
2409 
2410 	{"3DSTATE_SCISSOR_STATE_POINTERS", OP_3DSTATE_SCISSOR_STATE_POINTERS,
2411 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2412 
2413 	{"3DSTATE_GS", OP_3DSTATE_GS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2414 
2415 	{"3DSTATE_CLIP", OP_3DSTATE_CLIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2416 
2417 	{"3DSTATE_WM", OP_3DSTATE_WM, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2418 
2419 	{"3DSTATE_CONSTANT_GS", OP_3DSTATE_CONSTANT_GS, F_LEN_VAR, R_RCS,
2420 		D_BDW_PLUS, 0, 8, NULL},
2421 
2422 	{"3DSTATE_CONSTANT_PS", OP_3DSTATE_CONSTANT_PS, F_LEN_VAR, R_RCS,
2423 		D_BDW_PLUS, 0, 8, NULL},
2424 
2425 	{"3DSTATE_SAMPLE_MASK", OP_3DSTATE_SAMPLE_MASK, F_LEN_VAR, R_RCS,
2426 		D_ALL, 0, 8, NULL},
2427 
2428 	{"3DSTATE_CONSTANT_HS", OP_3DSTATE_CONSTANT_HS, F_LEN_VAR, R_RCS,
2429 		D_BDW_PLUS, 0, 8, NULL},
2430 
2431 	{"3DSTATE_CONSTANT_DS", OP_3DSTATE_CONSTANT_DS, F_LEN_VAR, R_RCS,
2432 		D_BDW_PLUS, 0, 8, NULL},
2433 
2434 	{"3DSTATE_HS", OP_3DSTATE_HS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2435 
2436 	{"3DSTATE_TE", OP_3DSTATE_TE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2437 
2438 	{"3DSTATE_DS", OP_3DSTATE_DS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2439 
2440 	{"3DSTATE_STREAMOUT", OP_3DSTATE_STREAMOUT, F_LEN_VAR, R_RCS,
2441 		D_ALL, 0, 8, NULL},
2442 
2443 	{"3DSTATE_SBE", OP_3DSTATE_SBE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2444 
2445 	{"3DSTATE_PS", OP_3DSTATE_PS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2446 
2447 	{"3DSTATE_DRAWING_RECTANGLE", OP_3DSTATE_DRAWING_RECTANGLE, F_LEN_VAR,
2448 		R_RCS, D_ALL, 0, 8, NULL},
2449 
2450 	{"3DSTATE_SAMPLER_PALETTE_LOAD0", OP_3DSTATE_SAMPLER_PALETTE_LOAD0,
2451 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2452 
2453 	{"3DSTATE_CHROMA_KEY", OP_3DSTATE_CHROMA_KEY, F_LEN_VAR, R_RCS, D_ALL,
2454 		0, 8, NULL},
2455 
2456 	{"3DSTATE_DEPTH_BUFFER", OP_3DSTATE_DEPTH_BUFFER, F_LEN_VAR, R_RCS,
2457 		D_ALL, ADDR_FIX_1(2), 8, NULL},
2458 
2459 	{"3DSTATE_POLY_STIPPLE_OFFSET", OP_3DSTATE_POLY_STIPPLE_OFFSET,
2460 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2461 
2462 	{"3DSTATE_POLY_STIPPLE_PATTERN", OP_3DSTATE_POLY_STIPPLE_PATTERN,
2463 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2464 
2465 	{"3DSTATE_LINE_STIPPLE", OP_3DSTATE_LINE_STIPPLE, F_LEN_VAR, R_RCS,
2466 		D_ALL, 0, 8, NULL},
2467 
2468 	{"3DSTATE_AA_LINE_PARAMS", OP_3DSTATE_AA_LINE_PARAMS, F_LEN_VAR, R_RCS,
2469 		D_ALL, 0, 8, NULL},
2470 
2471 	{"3DSTATE_GS_SVB_INDEX", OP_3DSTATE_GS_SVB_INDEX, F_LEN_VAR, R_RCS,
2472 		D_ALL, 0, 8, NULL},
2473 
2474 	{"3DSTATE_SAMPLER_PALETTE_LOAD1", OP_3DSTATE_SAMPLER_PALETTE_LOAD1,
2475 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2476 
2477 	{"3DSTATE_MULTISAMPLE", OP_3DSTATE_MULTISAMPLE_BDW, F_LEN_VAR, R_RCS,
2478 		D_BDW_PLUS, 0, 8, NULL},
2479 
2480 	{"3DSTATE_STENCIL_BUFFER", OP_3DSTATE_STENCIL_BUFFER, F_LEN_VAR, R_RCS,
2481 		D_ALL, ADDR_FIX_1(2), 8, NULL},
2482 
2483 	{"3DSTATE_HIER_DEPTH_BUFFER", OP_3DSTATE_HIER_DEPTH_BUFFER, F_LEN_VAR,
2484 		R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL},
2485 
2486 	{"3DSTATE_CLEAR_PARAMS", OP_3DSTATE_CLEAR_PARAMS, F_LEN_VAR,
2487 		R_RCS, D_ALL, 0, 8, NULL},
2488 
2489 	{"3DSTATE_PUSH_CONSTANT_ALLOC_VS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS,
2490 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2491 
2492 	{"3DSTATE_PUSH_CONSTANT_ALLOC_HS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS,
2493 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2494 
2495 	{"3DSTATE_PUSH_CONSTANT_ALLOC_DS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS,
2496 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2497 
2498 	{"3DSTATE_PUSH_CONSTANT_ALLOC_GS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS,
2499 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2500 
2501 	{"3DSTATE_PUSH_CONSTANT_ALLOC_PS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS,
2502 		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2503 
2504 	{"3DSTATE_MONOFILTER_SIZE", OP_3DSTATE_MONOFILTER_SIZE, F_LEN_VAR,
2505 		R_RCS, D_ALL, 0, 8, NULL},
2506 
2507 	{"3DSTATE_SO_DECL_LIST", OP_3DSTATE_SO_DECL_LIST, F_LEN_VAR, R_RCS,
2508 		D_ALL, 0, 9, NULL},
2509 
2510 	{"3DSTATE_SO_BUFFER", OP_3DSTATE_SO_BUFFER, F_LEN_VAR, R_RCS, D_BDW_PLUS,
2511 		ADDR_FIX_2(2, 4), 8, NULL},
2512 
2513 	{"3DSTATE_BINDING_TABLE_POOL_ALLOC",
2514 		OP_3DSTATE_BINDING_TABLE_POOL_ALLOC,
2515 		F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
2516 
2517 	{"3DSTATE_GATHER_POOL_ALLOC", OP_3DSTATE_GATHER_POOL_ALLOC,
2518 		F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
2519 
2520 	{"3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC",
2521 		OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC,
2522 		F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
2523 
2524 	{"3DSTATE_SAMPLE_PATTERN", OP_3DSTATE_SAMPLE_PATTERN, F_LEN_VAR, R_RCS,
2525 		D_BDW_PLUS, 0, 8, NULL},
2526 
2527 	{"PIPE_CONTROL", OP_PIPE_CONTROL, F_LEN_VAR, R_RCS, D_ALL,
2528 		ADDR_FIX_1(2), 8, cmd_handler_pipe_control},
2529 
2530 	{"3DPRIMITIVE", OP_3DPRIMITIVE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2531 
2532 	{"PIPELINE_SELECT", OP_PIPELINE_SELECT, F_LEN_CONST, R_RCS, D_ALL, 0,
2533 		1, NULL},
2534 
2535 	{"STATE_PREFETCH", OP_STATE_PREFETCH, F_LEN_VAR, R_RCS, D_ALL,
2536 		ADDR_FIX_1(1), 8, NULL},
2537 
2538 	{"STATE_SIP", OP_STATE_SIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2539 
2540 	{"STATE_BASE_ADDRESS", OP_STATE_BASE_ADDRESS, F_LEN_VAR, R_RCS, D_BDW_PLUS,
2541 		ADDR_FIX_5(1, 3, 4, 5, 6), 8, NULL},
2542 
2543 	{"OP_3D_MEDIA_0_1_4", OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL,
2544 		ADDR_FIX_1(1), 8, NULL},
2545 
2546 	{"OP_SWTESS_BASE_ADDRESS", OP_SWTESS_BASE_ADDRESS,
2547 		F_LEN_VAR, R_RCS, D_ALL, ADDR_FIX_2(1, 2), 3, NULL},
2548 
2549 	{"3DSTATE_VS", OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2550 
2551 	{"3DSTATE_SF", OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
2552 
2553 	{"3DSTATE_CONSTANT_VS", OP_3DSTATE_CONSTANT_VS, F_LEN_VAR, R_RCS, D_BDW_PLUS,
2554 		0, 8, NULL},
2555 
2556 	{"3DSTATE_COMPONENT_PACKING", OP_3DSTATE_COMPONENT_PACKING, F_LEN_VAR, R_RCS,
2557 		D_SKL_PLUS, 0, 8, NULL},
2558 
2559 	{"MEDIA_INTERFACE_DESCRIPTOR_LOAD", OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
2560 		F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL},
2561 
2562 	{"MEDIA_GATEWAY_STATE", OP_MEDIA_GATEWAY_STATE, F_LEN_VAR, R_RCS, D_ALL,
2563 		0, 16, NULL},
2564 
2565 	{"MEDIA_STATE_FLUSH", OP_MEDIA_STATE_FLUSH, F_LEN_VAR, R_RCS, D_ALL,
2566 		0, 16, NULL},
2567 
2568 	{"MEDIA_POOL_STATE", OP_MEDIA_POOL_STATE, F_LEN_VAR, R_RCS, D_ALL,
2569 		0, 16, NULL},
2570 
2571 	{"MEDIA_OBJECT", OP_MEDIA_OBJECT, F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL},
2572 
2573 	{"MEDIA_CURBE_LOAD", OP_MEDIA_CURBE_LOAD, F_LEN_VAR, R_RCS, D_ALL,
2574 		0, 16, NULL},
2575 
2576 	{"MEDIA_OBJECT_PRT", OP_MEDIA_OBJECT_PRT, F_LEN_VAR, R_RCS, D_ALL,
2577 		0, 16, NULL},
2578 
2579 	{"MEDIA_OBJECT_WALKER", OP_MEDIA_OBJECT_WALKER, F_LEN_VAR, R_RCS, D_ALL,
2580 		0, 16, NULL},
2581 
2582 	{"GPGPU_WALKER", OP_GPGPU_WALKER, F_LEN_VAR, R_RCS, D_ALL,
2583 		0, 8, NULL},
2584 
2585 	{"MEDIA_VFE_STATE", OP_MEDIA_VFE_STATE, F_LEN_VAR, R_RCS, D_ALL, 0, 16,
2586 		NULL},
2587 
2588 	{"3DSTATE_VF_STATISTICS_GM45", OP_3DSTATE_VF_STATISTICS_GM45,
2589 		F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
2590 
2591 	{"MFX_PIPE_MODE_SELECT", OP_MFX_PIPE_MODE_SELECT, F_LEN_VAR,
2592 		R_VCS, D_ALL, 0, 12, NULL},
2593 
2594 	{"MFX_SURFACE_STATE", OP_MFX_SURFACE_STATE, F_LEN_VAR,
2595 		R_VCS, D_ALL, 0, 12, NULL},
2596 
2597 	{"MFX_PIPE_BUF_ADDR_STATE", OP_MFX_PIPE_BUF_ADDR_STATE, F_LEN_VAR,
2598 		R_VCS, D_BDW_PLUS, 0, 12, NULL},
2599 
2600 	{"MFX_IND_OBJ_BASE_ADDR_STATE", OP_MFX_IND_OBJ_BASE_ADDR_STATE,
2601 		F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL},
2602 
2603 	{"MFX_BSP_BUF_BASE_ADDR_STATE", OP_MFX_BSP_BUF_BASE_ADDR_STATE,
2604 		F_LEN_VAR, R_VCS, D_BDW_PLUS, ADDR_FIX_3(1, 3, 5), 12, NULL},
2605 
2606 	{"OP_2_0_0_5", OP_2_0_0_5, F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL},
2607 
2608 	{"MFX_STATE_POINTER", OP_MFX_STATE_POINTER, F_LEN_VAR,
2609 		R_VCS, D_ALL, 0, 12, NULL},
2610 
2611 	{"MFX_QM_STATE", OP_MFX_QM_STATE, F_LEN_VAR,
2612 		R_VCS, D_ALL, 0, 12, NULL},
2613 
2614 	{"MFX_FQM_STATE", OP_MFX_FQM_STATE, F_LEN_VAR,
2615 		R_VCS, D_ALL, 0, 12, NULL},
2616 
2617 	{"MFX_PAK_INSERT_OBJECT", OP_MFX_PAK_INSERT_OBJECT, F_LEN_VAR,
2618 		R_VCS, D_ALL, 0, 12, NULL},
2619 
2620 	{"MFX_STITCH_OBJECT", OP_MFX_STITCH_OBJECT, F_LEN_VAR,
2621 		R_VCS, D_ALL, 0, 12, NULL},
2622 
2623 	{"MFD_IT_OBJECT", OP_MFD_IT_OBJECT, F_LEN_VAR,
2624 		R_VCS, D_ALL, 0, 12, NULL},
2625 
2626 	{"MFX_WAIT", OP_MFX_WAIT, F_LEN_VAR,
2627 		R_VCS, D_ALL, 0, 6, NULL},
2628 
2629 	{"MFX_AVC_IMG_STATE", OP_MFX_AVC_IMG_STATE, F_LEN_VAR,
2630 		R_VCS, D_ALL, 0, 12, NULL},
2631 
2632 	{"MFX_AVC_QM_STATE", OP_MFX_AVC_QM_STATE, F_LEN_VAR,
2633 		R_VCS, D_ALL, 0, 12, NULL},
2634 
2635 	{"MFX_AVC_DIRECTMODE_STATE", OP_MFX_AVC_DIRECTMODE_STATE, F_LEN_VAR,
2636 		R_VCS, D_ALL, 0, 12, NULL},
2637 
2638 	{"MFX_AVC_SLICE_STATE", OP_MFX_AVC_SLICE_STATE, F_LEN_VAR,
2639 		R_VCS, D_ALL, 0, 12, NULL},
2640 
2641 	{"MFX_AVC_REF_IDX_STATE", OP_MFX_AVC_REF_IDX_STATE, F_LEN_VAR,
2642 		R_VCS, D_ALL, 0, 12, NULL},
2643 
2644 	{"MFX_AVC_WEIGHTOFFSET_STATE", OP_MFX_AVC_WEIGHTOFFSET_STATE, F_LEN_VAR,
2645 		R_VCS, D_ALL, 0, 12, NULL},
2646 
2647 	{"MFD_AVC_PICID_STATE", OP_MFD_AVC_PICID_STATE, F_LEN_VAR,
2648 		R_VCS, D_ALL, 0, 12, NULL},
2649 	{"MFD_AVC_DPB_STATE", OP_MFD_AVC_DPB_STATE, F_LEN_VAR,
2650 		R_VCS, D_ALL, 0, 12, NULL},
2651 
2652 	{"MFD_AVC_BSD_OBJECT", OP_MFD_AVC_BSD_OBJECT, F_LEN_VAR,
2653 		R_VCS, D_ALL, 0, 12, NULL},
2654 
2655 	{"MFD_AVC_SLICEADDR", OP_MFD_AVC_SLICEADDR, F_LEN_VAR,
2656 		R_VCS, D_ALL, ADDR_FIX_1(2), 12, NULL},
2657 
2658 	{"MFC_AVC_PAK_OBJECT", OP_MFC_AVC_PAK_OBJECT, F_LEN_VAR,
2659 		R_VCS, D_ALL, 0, 12, NULL},
2660 
2661 	{"MFX_VC1_PRED_PIPE_STATE", OP_MFX_VC1_PRED_PIPE_STATE, F_LEN_VAR,
2662 		R_VCS, D_ALL, 0, 12, NULL},
2663 
2664 	{"MFX_VC1_DIRECTMODE_STATE", OP_MFX_VC1_DIRECTMODE_STATE, F_LEN_VAR,
2665 		R_VCS, D_ALL, 0, 12, NULL},
2666 
2667 	{"MFD_VC1_SHORT_PIC_STATE", OP_MFD_VC1_SHORT_PIC_STATE, F_LEN_VAR,
2668 		R_VCS, D_ALL, 0, 12, NULL},
2669 
2670 	{"MFD_VC1_LONG_PIC_STATE", OP_MFD_VC1_LONG_PIC_STATE, F_LEN_VAR,
2671 		R_VCS, D_ALL, 0, 12, NULL},
2672 
2673 	{"MFD_VC1_BSD_OBJECT", OP_MFD_VC1_BSD_OBJECT, F_LEN_VAR,
2674 		R_VCS, D_ALL, 0, 12, NULL},
2675 
2676 	{"MFC_MPEG2_SLICEGROUP_STATE", OP_MFC_MPEG2_SLICEGROUP_STATE, F_LEN_VAR,
2677 		R_VCS, D_ALL, 0, 12, NULL},
2678 
2679 	{"MFC_MPEG2_PAK_OBJECT", OP_MFC_MPEG2_PAK_OBJECT, F_LEN_VAR,
2680 		R_VCS, D_ALL, 0, 12, NULL},
2681 
2682 	{"MFX_MPEG2_PIC_STATE", OP_MFX_MPEG2_PIC_STATE, F_LEN_VAR,
2683 		R_VCS, D_ALL, 0, 12, NULL},
2684 
2685 	{"MFX_MPEG2_QM_STATE", OP_MFX_MPEG2_QM_STATE, F_LEN_VAR,
2686 		R_VCS, D_ALL, 0, 12, NULL},
2687 
2688 	{"MFD_MPEG2_BSD_OBJECT", OP_MFD_MPEG2_BSD_OBJECT, F_LEN_VAR,
2689 		R_VCS, D_ALL, 0, 12, NULL},
2690 
2691 	{"MFX_2_6_0_0", OP_MFX_2_6_0_0, F_LEN_VAR, R_VCS, D_ALL,
2692 		0, 16, NULL},
2693 
2694 	{"MFX_2_6_0_9", OP_MFX_2_6_0_9, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL},
2695 
2696 	{"MFX_2_6_0_8", OP_MFX_2_6_0_8, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL},
2697 
2698 	{"MFX_JPEG_PIC_STATE", OP_MFX_JPEG_PIC_STATE, F_LEN_VAR,
2699 		R_VCS, D_ALL, 0, 12, NULL},
2700 
2701 	{"MFX_JPEG_HUFF_TABLE_STATE", OP_MFX_JPEG_HUFF_TABLE_STATE, F_LEN_VAR,
2702 		R_VCS, D_ALL, 0, 12, NULL},
2703 
2704 	{"MFD_JPEG_BSD_OBJECT", OP_MFD_JPEG_BSD_OBJECT, F_LEN_VAR,
2705 		R_VCS, D_ALL, 0, 12, NULL},
2706 
2707 	{"VEBOX_STATE", OP_VEB_STATE, F_LEN_VAR, R_VECS, D_ALL, 0, 12, NULL},
2708 
2709 	{"VEBOX_SURFACE_STATE", OP_VEB_SURFACE_STATE, F_LEN_VAR, R_VECS, D_ALL,
2710 		0, 12, NULL},
2711 
2712 	{"VEB_DI_IECP", OP_VEB_DNDI_IECP_STATE, F_LEN_VAR, R_VECS, D_BDW_PLUS,
2713 		0, 12, NULL},
2714 };
2715 
add_cmd_entry(struct intel_gvt * gvt,struct cmd_entry * e)2716 static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e)
2717 {
2718 	hash_add(gvt->cmd_table, &e->hlist, e->info->opcode);
2719 }
2720 
2721 /* call the cmd handler, and advance ip */
cmd_parser_exec(struct parser_exec_state * s)2722 static int cmd_parser_exec(struct parser_exec_state *s)
2723 {
2724 	struct intel_vgpu *vgpu = s->vgpu;
2725 	const struct cmd_info *info;
2726 	u32 cmd;
2727 	int ret = 0;
2728 
2729 	cmd = cmd_val(s, 0);
2730 
2731 	/* fastpath for MI_NOOP */
2732 	if (cmd == MI_NOOP)
2733 		info = &cmd_info[mi_noop_index];
2734 	else
2735 		info = get_cmd_info(s->vgpu->gvt, cmd, s->engine);
2736 
2737 	if (info == NULL) {
2738 		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %s, workload=%p\n",
2739 			     cmd, get_opcode(cmd, s->engine),
2740 			     repr_addr_type(s->buf_addr_type),
2741 			     s->engine->name, s->workload);
2742 		return -EBADRQC;
2743 	}
2744 
2745 	s->info = info;
2746 
2747 	trace_gvt_command(vgpu->id, s->engine->id, s->ip_gma, s->ip_va,
2748 			  cmd_length(s), s->buf_type, s->buf_addr_type,
2749 			  s->workload, info->name);
2750 
2751 	if ((info->flag & F_LEN_MASK) == F_LEN_VAR_FIXED) {
2752 		ret = gvt_check_valid_cmd_length(cmd_length(s),
2753 						 info->valid_len);
2754 		if (ret)
2755 			return ret;
2756 	}
2757 
2758 	if (info->handler) {
2759 		ret = info->handler(s);
2760 		if (ret < 0) {
2761 			gvt_vgpu_err("%s handler error\n", info->name);
2762 			return ret;
2763 		}
2764 	}
2765 
2766 	if (!(info->flag & F_IP_ADVANCE_CUSTOM)) {
2767 		ret = cmd_advance_default(s);
2768 		if (ret) {
2769 			gvt_vgpu_err("%s IP advance error\n", info->name);
2770 			return ret;
2771 		}
2772 	}
2773 	return 0;
2774 }
2775 
gma_out_of_range(unsigned long gma,unsigned long gma_head,unsigned int gma_tail)2776 static inline bool gma_out_of_range(unsigned long gma,
2777 		unsigned long gma_head, unsigned int gma_tail)
2778 {
2779 	if (gma_tail >= gma_head)
2780 		return (gma < gma_head) || (gma > gma_tail);
2781 	else
2782 		return (gma > gma_tail) && (gma < gma_head);
2783 }
2784 
2785 /* Keep the consistent return type, e.g EBADRQC for unknown
2786  * cmd, EFAULT for invalid address, EPERM for nonpriv. later
2787  * works as the input of VM healthy status.
2788  */
command_scan(struct parser_exec_state * s,unsigned long rb_head,unsigned long rb_tail,unsigned long rb_start,unsigned long rb_len)2789 static int command_scan(struct parser_exec_state *s,
2790 		unsigned long rb_head, unsigned long rb_tail,
2791 		unsigned long rb_start, unsigned long rb_len)
2792 {
2793 
2794 	unsigned long gma_head, gma_tail, gma_bottom;
2795 	int ret = 0;
2796 	struct intel_vgpu *vgpu = s->vgpu;
2797 
2798 	gma_head = rb_start + rb_head;
2799 	gma_tail = rb_start + rb_tail;
2800 	gma_bottom = rb_start +  rb_len;
2801 
2802 	while (s->ip_gma != gma_tail) {
2803 		if (s->buf_type == RING_BUFFER_INSTRUCTION ||
2804 				s->buf_type == RING_BUFFER_CTX) {
2805 			if (!(s->ip_gma >= rb_start) ||
2806 				!(s->ip_gma < gma_bottom)) {
2807 				gvt_vgpu_err("ip_gma %lx out of ring scope."
2808 					"(base:0x%lx, bottom: 0x%lx)\n",
2809 					s->ip_gma, rb_start,
2810 					gma_bottom);
2811 				parser_exec_state_dump(s);
2812 				return -EFAULT;
2813 			}
2814 			if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) {
2815 				gvt_vgpu_err("ip_gma %lx out of range."
2816 					"base 0x%lx head 0x%lx tail 0x%lx\n",
2817 					s->ip_gma, rb_start,
2818 					rb_head, rb_tail);
2819 				parser_exec_state_dump(s);
2820 				break;
2821 			}
2822 		}
2823 		ret = cmd_parser_exec(s);
2824 		if (ret) {
2825 			gvt_vgpu_err("cmd parser error\n");
2826 			parser_exec_state_dump(s);
2827 			break;
2828 		}
2829 	}
2830 
2831 	return ret;
2832 }
2833 
scan_workload(struct intel_vgpu_workload * workload)2834 static int scan_workload(struct intel_vgpu_workload *workload)
2835 {
2836 	unsigned long gma_head, gma_tail;
2837 	struct parser_exec_state s;
2838 	int ret = 0;
2839 
2840 	/* ring base is page aligned */
2841 	if (WARN_ON(!IS_ALIGNED(workload->rb_start, I915_GTT_PAGE_SIZE)))
2842 		return -EINVAL;
2843 
2844 	gma_head = workload->rb_start + workload->rb_head;
2845 	gma_tail = workload->rb_start + workload->rb_tail;
2846 
2847 	s.buf_type = RING_BUFFER_INSTRUCTION;
2848 	s.buf_addr_type = GTT_BUFFER;
2849 	s.vgpu = workload->vgpu;
2850 	s.engine = workload->engine;
2851 	s.ring_start = workload->rb_start;
2852 	s.ring_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
2853 	s.ring_head = gma_head;
2854 	s.ring_tail = gma_tail;
2855 	s.rb_va = workload->shadow_ring_buffer_va;
2856 	s.workload = workload;
2857 	s.is_ctx_wa = false;
2858 
2859 	if (bypass_scan_mask & workload->engine->mask || gma_head == gma_tail)
2860 		return 0;
2861 
2862 	ret = ip_gma_set(&s, gma_head);
2863 	if (ret)
2864 		goto out;
2865 
2866 	ret = command_scan(&s, workload->rb_head, workload->rb_tail,
2867 		workload->rb_start, _RING_CTL_BUF_SIZE(workload->rb_ctl));
2868 
2869 out:
2870 	return ret;
2871 }
2872 
scan_wa_ctx(struct intel_shadow_wa_ctx * wa_ctx)2873 static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
2874 {
2875 
2876 	unsigned long gma_head, gma_tail, ring_size, ring_tail;
2877 	struct parser_exec_state s;
2878 	int ret = 0;
2879 	struct intel_vgpu_workload *workload = container_of(wa_ctx,
2880 				struct intel_vgpu_workload,
2881 				wa_ctx);
2882 
2883 	/* ring base is page aligned */
2884 	if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma,
2885 					I915_GTT_PAGE_SIZE)))
2886 		return -EINVAL;
2887 
2888 	ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(u32);
2889 	ring_size = round_up(wa_ctx->indirect_ctx.size + CACHELINE_BYTES,
2890 			PAGE_SIZE);
2891 	gma_head = wa_ctx->indirect_ctx.guest_gma;
2892 	gma_tail = wa_ctx->indirect_ctx.guest_gma + ring_tail;
2893 
2894 	s.buf_type = RING_BUFFER_INSTRUCTION;
2895 	s.buf_addr_type = GTT_BUFFER;
2896 	s.vgpu = workload->vgpu;
2897 	s.engine = workload->engine;
2898 	s.ring_start = wa_ctx->indirect_ctx.guest_gma;
2899 	s.ring_size = ring_size;
2900 	s.ring_head = gma_head;
2901 	s.ring_tail = gma_tail;
2902 	s.rb_va = wa_ctx->indirect_ctx.shadow_va;
2903 	s.workload = workload;
2904 	s.is_ctx_wa = true;
2905 
2906 	ret = ip_gma_set(&s, gma_head);
2907 	if (ret)
2908 		goto out;
2909 
2910 	ret = command_scan(&s, 0, ring_tail,
2911 		wa_ctx->indirect_ctx.guest_gma, ring_size);
2912 out:
2913 	return ret;
2914 }
2915 
shadow_workload_ring_buffer(struct intel_vgpu_workload * workload)2916 static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
2917 {
2918 	struct intel_vgpu *vgpu = workload->vgpu;
2919 	struct intel_vgpu_submission *s = &vgpu->submission;
2920 	unsigned long gma_head, gma_tail, gma_top, guest_rb_size;
2921 	void *shadow_ring_buffer_va;
2922 	int ret;
2923 
2924 	guest_rb_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
2925 
2926 	/* calculate workload ring buffer size */
2927 	workload->rb_len = (workload->rb_tail + guest_rb_size -
2928 			workload->rb_head) % guest_rb_size;
2929 
2930 	gma_head = workload->rb_start + workload->rb_head;
2931 	gma_tail = workload->rb_start + workload->rb_tail;
2932 	gma_top = workload->rb_start + guest_rb_size;
2933 
2934 	if (workload->rb_len > s->ring_scan_buffer_size[workload->engine->id]) {
2935 		void *p;
2936 
2937 		/* realloc the new ring buffer if needed */
2938 		p = krealloc(s->ring_scan_buffer[workload->engine->id],
2939 			     workload->rb_len, GFP_KERNEL);
2940 		if (!p) {
2941 			gvt_vgpu_err("fail to re-alloc ring scan buffer\n");
2942 			return -ENOMEM;
2943 		}
2944 		s->ring_scan_buffer[workload->engine->id] = p;
2945 		s->ring_scan_buffer_size[workload->engine->id] = workload->rb_len;
2946 	}
2947 
2948 	shadow_ring_buffer_va = s->ring_scan_buffer[workload->engine->id];
2949 
2950 	/* get shadow ring buffer va */
2951 	workload->shadow_ring_buffer_va = shadow_ring_buffer_va;
2952 
2953 	/* head > tail --> copy head <-> top */
2954 	if (gma_head > gma_tail) {
2955 		ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm,
2956 				      gma_head, gma_top, shadow_ring_buffer_va);
2957 		if (ret < 0) {
2958 			gvt_vgpu_err("fail to copy guest ring buffer\n");
2959 			return ret;
2960 		}
2961 		shadow_ring_buffer_va += ret;
2962 		gma_head = workload->rb_start;
2963 	}
2964 
2965 	/* copy head or start <-> tail */
2966 	ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm, gma_head, gma_tail,
2967 				shadow_ring_buffer_va);
2968 	if (ret < 0) {
2969 		gvt_vgpu_err("fail to copy guest ring buffer\n");
2970 		return ret;
2971 	}
2972 	return 0;
2973 }
2974 
intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload * workload)2975 int intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload *workload)
2976 {
2977 	int ret;
2978 	struct intel_vgpu *vgpu = workload->vgpu;
2979 
2980 	ret = shadow_workload_ring_buffer(workload);
2981 	if (ret) {
2982 		gvt_vgpu_err("fail to shadow workload ring_buffer\n");
2983 		return ret;
2984 	}
2985 
2986 	ret = scan_workload(workload);
2987 	if (ret) {
2988 		gvt_vgpu_err("scan workload error\n");
2989 		return ret;
2990 	}
2991 	return 0;
2992 }
2993 
shadow_indirect_ctx(struct intel_shadow_wa_ctx * wa_ctx)2994 static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx)
2995 {
2996 	int ctx_size = wa_ctx->indirect_ctx.size;
2997 	unsigned long guest_gma = wa_ctx->indirect_ctx.guest_gma;
2998 	struct intel_vgpu_workload *workload = container_of(wa_ctx,
2999 					struct intel_vgpu_workload,
3000 					wa_ctx);
3001 	struct intel_vgpu *vgpu = workload->vgpu;
3002 	struct drm_i915_gem_object *obj;
3003 	int ret = 0;
3004 	void *map;
3005 
3006 	obj = i915_gem_object_create_shmem(workload->engine->i915,
3007 					   roundup(ctx_size + CACHELINE_BYTES,
3008 						   PAGE_SIZE));
3009 	if (IS_ERR(obj))
3010 		return PTR_ERR(obj);
3011 
3012 	/* get the va of the shadow batch buffer */
3013 	map = i915_gem_object_pin_map(obj, I915_MAP_WB);
3014 	if (IS_ERR(map)) {
3015 		gvt_vgpu_err("failed to vmap shadow indirect ctx\n");
3016 		ret = PTR_ERR(map);
3017 		goto put_obj;
3018 	}
3019 
3020 	i915_gem_object_lock(obj, NULL);
3021 	ret = i915_gem_object_set_to_cpu_domain(obj, false);
3022 	i915_gem_object_unlock(obj);
3023 	if (ret) {
3024 		gvt_vgpu_err("failed to set shadow indirect ctx to CPU\n");
3025 		goto unmap_src;
3026 	}
3027 
3028 	ret = copy_gma_to_hva(workload->vgpu,
3029 				workload->vgpu->gtt.ggtt_mm,
3030 				guest_gma, guest_gma + ctx_size,
3031 				map);
3032 	if (ret < 0) {
3033 		gvt_vgpu_err("fail to copy guest indirect ctx\n");
3034 		goto unmap_src;
3035 	}
3036 
3037 	wa_ctx->indirect_ctx.obj = obj;
3038 	wa_ctx->indirect_ctx.shadow_va = map;
3039 	return 0;
3040 
3041 unmap_src:
3042 	i915_gem_object_unpin_map(obj);
3043 put_obj:
3044 	i915_gem_object_put(obj);
3045 	return ret;
3046 }
3047 
combine_wa_ctx(struct intel_shadow_wa_ctx * wa_ctx)3048 static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
3049 {
3050 	u32 per_ctx_start[CACHELINE_DWORDS] = {0};
3051 	unsigned char *bb_start_sva;
3052 
3053 	if (!wa_ctx->per_ctx.valid)
3054 		return 0;
3055 
3056 	per_ctx_start[0] = 0x18800001;
3057 	per_ctx_start[1] = wa_ctx->per_ctx.guest_gma;
3058 
3059 	bb_start_sva = (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
3060 				wa_ctx->indirect_ctx.size;
3061 
3062 	memcpy(bb_start_sva, per_ctx_start, CACHELINE_BYTES);
3063 
3064 	return 0;
3065 }
3066 
intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx * wa_ctx)3067 int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
3068 {
3069 	int ret;
3070 	struct intel_vgpu_workload *workload = container_of(wa_ctx,
3071 					struct intel_vgpu_workload,
3072 					wa_ctx);
3073 	struct intel_vgpu *vgpu = workload->vgpu;
3074 
3075 	if (wa_ctx->indirect_ctx.size == 0)
3076 		return 0;
3077 
3078 	ret = shadow_indirect_ctx(wa_ctx);
3079 	if (ret) {
3080 		gvt_vgpu_err("fail to shadow indirect ctx\n");
3081 		return ret;
3082 	}
3083 
3084 	combine_wa_ctx(wa_ctx);
3085 
3086 	ret = scan_wa_ctx(wa_ctx);
3087 	if (ret) {
3088 		gvt_vgpu_err("scan wa ctx error\n");
3089 		return ret;
3090 	}
3091 
3092 	return 0;
3093 }
3094 
3095 /* generate dummy contexts by sending empty requests to HW, and let
3096  * the HW to fill Engine Contexts. This dummy contexts are used for
3097  * initialization purpose (update reg whitelist), so referred to as
3098  * init context here
3099  */
intel_gvt_update_reg_whitelist(struct intel_vgpu * vgpu)3100 void intel_gvt_update_reg_whitelist(struct intel_vgpu *vgpu)
3101 {
3102 	const unsigned long start = LRC_STATE_PN * PAGE_SIZE;
3103 	struct intel_gvt *gvt = vgpu->gvt;
3104 	struct intel_engine_cs *engine;
3105 	enum intel_engine_id id;
3106 
3107 	if (gvt->is_reg_whitelist_updated)
3108 		return;
3109 
3110 	/* scan init ctx to update cmd accessible list */
3111 	for_each_engine(engine, gvt->gt, id) {
3112 		struct parser_exec_state s;
3113 		void *vaddr;
3114 		int ret;
3115 
3116 		if (!engine->default_state)
3117 			continue;
3118 
3119 		vaddr = shmem_pin_map(engine->default_state);
3120 		if (!vaddr) {
3121 			gvt_err("failed to map %s->default state\n",
3122 				engine->name);
3123 			return;
3124 		}
3125 
3126 		s.buf_type = RING_BUFFER_CTX;
3127 		s.buf_addr_type = GTT_BUFFER;
3128 		s.vgpu = vgpu;
3129 		s.engine = engine;
3130 		s.ring_start = 0;
3131 		s.ring_size = engine->context_size - start;
3132 		s.ring_head = 0;
3133 		s.ring_tail = s.ring_size;
3134 		s.rb_va = vaddr + start;
3135 		s.workload = NULL;
3136 		s.is_ctx_wa = false;
3137 		s.is_init_ctx = true;
3138 
3139 		/* skipping the first RING_CTX_SIZE(0x50) dwords */
3140 		ret = ip_gma_set(&s, RING_CTX_SIZE);
3141 		if (ret == 0) {
3142 			ret = command_scan(&s, 0, s.ring_size, 0, s.ring_size);
3143 			if (ret)
3144 				gvt_err("Scan init ctx error\n");
3145 		}
3146 
3147 		shmem_unpin_map(engine->default_state, vaddr);
3148 		if (ret)
3149 			return;
3150 	}
3151 
3152 	gvt->is_reg_whitelist_updated = true;
3153 }
3154 
intel_gvt_scan_engine_context(struct intel_vgpu_workload * workload)3155 int intel_gvt_scan_engine_context(struct intel_vgpu_workload *workload)
3156 {
3157 	struct intel_vgpu *vgpu = workload->vgpu;
3158 	unsigned long gma_head, gma_tail, gma_start, ctx_size;
3159 	struct parser_exec_state s;
3160 	int ring_id = workload->engine->id;
3161 	struct intel_context *ce = vgpu->submission.shadow[ring_id];
3162 	int ret;
3163 
3164 	GEM_BUG_ON(atomic_read(&ce->pin_count) < 0);
3165 
3166 	ctx_size = workload->engine->context_size - PAGE_SIZE;
3167 
3168 	/* Only ring contxt is loaded to HW for inhibit context, no need to
3169 	 * scan engine context
3170 	 */
3171 	if (is_inhibit_context(ce))
3172 		return 0;
3173 
3174 	gma_start = i915_ggtt_offset(ce->state) + LRC_STATE_PN*PAGE_SIZE;
3175 	gma_head = 0;
3176 	gma_tail = ctx_size;
3177 
3178 	s.buf_type = RING_BUFFER_CTX;
3179 	s.buf_addr_type = GTT_BUFFER;
3180 	s.vgpu = workload->vgpu;
3181 	s.engine = workload->engine;
3182 	s.ring_start = gma_start;
3183 	s.ring_size = ctx_size;
3184 	s.ring_head = gma_start + gma_head;
3185 	s.ring_tail = gma_start + gma_tail;
3186 	s.rb_va = ce->lrc_reg_state;
3187 	s.workload = workload;
3188 	s.is_ctx_wa = false;
3189 	s.is_init_ctx = false;
3190 
3191 	/* don't scan the first RING_CTX_SIZE(0x50) dwords, as it's ring
3192 	 * context
3193 	 */
3194 	ret = ip_gma_set(&s, gma_start + gma_head + RING_CTX_SIZE);
3195 	if (ret)
3196 		goto out;
3197 
3198 	ret = command_scan(&s, gma_head, gma_tail,
3199 		gma_start, ctx_size);
3200 out:
3201 	if (ret)
3202 		gvt_vgpu_err("scan shadow ctx error\n");
3203 
3204 	return ret;
3205 }
3206 
init_cmd_table(struct intel_gvt * gvt)3207 static int init_cmd_table(struct intel_gvt *gvt)
3208 {
3209 	unsigned int gen_type = intel_gvt_get_device_type(gvt);
3210 	int i;
3211 
3212 	for (i = 0; i < ARRAY_SIZE(cmd_info); i++) {
3213 		struct cmd_entry *e;
3214 
3215 		if (!(cmd_info[i].devices & gen_type))
3216 			continue;
3217 
3218 		e = kzalloc(sizeof(*e), GFP_KERNEL);
3219 		if (!e)
3220 			return -ENOMEM;
3221 
3222 		e->info = &cmd_info[i];
3223 		if (cmd_info[i].opcode == OP_MI_NOOP)
3224 			mi_noop_index = i;
3225 
3226 		INIT_HLIST_NODE(&e->hlist);
3227 		add_cmd_entry(gvt, e);
3228 		gvt_dbg_cmd("add %-30s op %04x flag %x devs %02x rings %02x\n",
3229 			    e->info->name, e->info->opcode, e->info->flag,
3230 			    e->info->devices, e->info->rings);
3231 	}
3232 
3233 	return 0;
3234 }
3235 
clean_cmd_table(struct intel_gvt * gvt)3236 static void clean_cmd_table(struct intel_gvt *gvt)
3237 {
3238 	struct hlist_node *tmp;
3239 	struct cmd_entry *e;
3240 	int i;
3241 
3242 	hash_for_each_safe(gvt->cmd_table, i, tmp, e, hlist)
3243 		kfree(e);
3244 
3245 	hash_init(gvt->cmd_table);
3246 }
3247 
intel_gvt_clean_cmd_parser(struct intel_gvt * gvt)3248 void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt)
3249 {
3250 	clean_cmd_table(gvt);
3251 }
3252 
intel_gvt_init_cmd_parser(struct intel_gvt * gvt)3253 int intel_gvt_init_cmd_parser(struct intel_gvt *gvt)
3254 {
3255 	int ret;
3256 
3257 	ret = init_cmd_table(gvt);
3258 	if (ret) {
3259 		intel_gvt_clean_cmd_parser(gvt);
3260 		return ret;
3261 	}
3262 	return 0;
3263 }
3264