1 /*
2  * Copyright © 2014 Broadcom
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 /**
25  * DOC: Shader validator for VC4.
26  *
27  * Since the VC4 has no IOMMU between it and system memory, a user
28  * with access to execute shaders could escalate privilege by
29  * overwriting system memory (using the VPM write address register in
30  * the general-purpose DMA mode) or reading system memory it shouldn't
31  * (reading it as a texture, uniform data, or direct-addressed TMU
32  * lookup).
33  *
34  * The shader validator walks over a shader's BO, ensuring that its
35  * accesses are appropriately bounded, and recording where texture
36  * accesses are made so that we can do relocations for them in the
37  * uniform stream.
38  *
39  * Shader BO are immutable for their lifetimes (enforced by not
40  * allowing mmaps, GEM prime export, or rendering to from a CL), so
41  * this validation is only performed at BO creation time.
42  */
43 
44 #include "vc4_drv.h"
45 #include "vc4_qpu_defines.h"
46 
47 #define LIVE_REG_COUNT (32 + 32 + 4)
48 
49 struct vc4_shader_validation_state {
50 	/* Current IP being validated. */
51 	uint32_t ip;
52 
53 	/* IP at the end of the BO, do not read shader[max_ip] */
54 	uint32_t max_ip;
55 
56 	uint64_t *shader;
57 
58 	struct vc4_texture_sample_info tmu_setup[2];
59 	int tmu_write_count[2];
60 
61 	/* For registers that were last written to by a MIN instruction with
62 	 * one argument being a uniform, the address of the uniform.
63 	 * Otherwise, ~0.
64 	 *
65 	 * This is used for the validation of direct address memory reads.
66 	 */
67 	uint32_t live_min_clamp_offsets[LIVE_REG_COUNT];
68 	bool live_max_clamp_regs[LIVE_REG_COUNT];
69 	uint32_t live_immediates[LIVE_REG_COUNT];
70 
71 	/* Bitfield of which IPs are used as branch targets.
72 	 *
73 	 * Used for validation that the uniform stream is updated at the right
74 	 * points and clearing the texturing/clamping state.
75 	 */
76 	unsigned long *branch_targets;
77 
78 	/* Set when entering a basic block, and cleared when the uniform
79 	 * address update is found.  This is used to make sure that we don't
80 	 * read uniforms when the address is undefined.
81 	 */
82 	bool needs_uniform_address_update;
83 
84 	/* Set when we find a backwards branch.  If the branch is backwards,
85 	 * the taraget is probably doing an address reset to read uniforms,
86 	 * and so we need to be sure that a uniforms address is present in the
87 	 * stream, even if the shader didn't need to read uniforms in later
88 	 * basic blocks.
89 	 */
90 	bool needs_uniform_address_for_loop;
91 
92 	/* Set when we find an instruction writing the top half of the
93 	 * register files.  If we allowed writing the unusable regs in
94 	 * a threaded shader, then the other shader running on our
95 	 * QPU's clamp validation would be invalid.
96 	 */
97 	bool all_registers_used;
98 };
99 
100 static uint32_t
waddr_to_live_reg_index(uint32_t waddr,bool is_b)101 waddr_to_live_reg_index(uint32_t waddr, bool is_b)
102 {
103 	if (waddr < 32) {
104 		if (is_b)
105 			return 32 + waddr;
106 		else
107 			return waddr;
108 	} else if (waddr <= QPU_W_ACC3) {
109 		return 64 + waddr - QPU_W_ACC0;
110 	} else {
111 		return ~0;
112 	}
113 }
114 
115 static uint32_t
raddr_add_a_to_live_reg_index(uint64_t inst)116 raddr_add_a_to_live_reg_index(uint64_t inst)
117 {
118 	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
119 	uint32_t add_a = QPU_GET_FIELD(inst, QPU_ADD_A);
120 	uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
121 	uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
122 
123 	if (add_a == QPU_MUX_A)
124 		return raddr_a;
125 	else if (add_a == QPU_MUX_B && sig != QPU_SIG_SMALL_IMM)
126 		return 32 + raddr_b;
127 	else if (add_a <= QPU_MUX_R3)
128 		return 64 + add_a;
129 	else
130 		return ~0;
131 }
132 
133 static bool
live_reg_is_upper_half(uint32_t lri)134 live_reg_is_upper_half(uint32_t lri)
135 {
136 	return	(lri >= 16 && lri < 32) ||
137 		(lri >= 32 + 16 && lri < 32 + 32);
138 }
139 
140 static bool
is_tmu_submit(uint32_t waddr)141 is_tmu_submit(uint32_t waddr)
142 {
143 	return (waddr == QPU_W_TMU0_S ||
144 		waddr == QPU_W_TMU1_S);
145 }
146 
147 static bool
is_tmu_write(uint32_t waddr)148 is_tmu_write(uint32_t waddr)
149 {
150 	return (waddr >= QPU_W_TMU0_S &&
151 		waddr <= QPU_W_TMU1_B);
152 }
153 
154 static bool
record_texture_sample(struct vc4_validated_shader_info * validated_shader,struct vc4_shader_validation_state * validation_state,int tmu)155 record_texture_sample(struct vc4_validated_shader_info *validated_shader,
156 		      struct vc4_shader_validation_state *validation_state,
157 		      int tmu)
158 {
159 	uint32_t s = validated_shader->num_texture_samples;
160 	int i;
161 	struct vc4_texture_sample_info *temp_samples;
162 
163 	temp_samples = krealloc(validated_shader->texture_samples,
164 				(s + 1) * sizeof(*temp_samples),
165 				GFP_KERNEL);
166 	if (!temp_samples)
167 		return false;
168 
169 	memcpy(&temp_samples[s],
170 	       &validation_state->tmu_setup[tmu],
171 	       sizeof(*temp_samples));
172 
173 	validated_shader->num_texture_samples = s + 1;
174 	validated_shader->texture_samples = temp_samples;
175 
176 	for (i = 0; i < 4; i++)
177 		validation_state->tmu_setup[tmu].p_offset[i] = ~0;
178 
179 	return true;
180 }
181 
182 static bool
check_tmu_write(struct vc4_validated_shader_info * validated_shader,struct vc4_shader_validation_state * validation_state,bool is_mul)183 check_tmu_write(struct vc4_validated_shader_info *validated_shader,
184 		struct vc4_shader_validation_state *validation_state,
185 		bool is_mul)
186 {
187 	uint64_t inst = validation_state->shader[validation_state->ip];
188 	uint32_t waddr = (is_mul ?
189 			  QPU_GET_FIELD(inst, QPU_WADDR_MUL) :
190 			  QPU_GET_FIELD(inst, QPU_WADDR_ADD));
191 	uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
192 	uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
193 	int tmu = waddr > QPU_W_TMU0_B;
194 	bool submit = is_tmu_submit(waddr);
195 	bool is_direct = submit && validation_state->tmu_write_count[tmu] == 0;
196 	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
197 
198 	if (is_direct) {
199 		uint32_t add_b = QPU_GET_FIELD(inst, QPU_ADD_B);
200 		uint32_t clamp_reg, clamp_offset;
201 
202 		if (sig == QPU_SIG_SMALL_IMM) {
203 			DRM_DEBUG("direct TMU read used small immediate\n");
204 			return false;
205 		}
206 
207 		/* Make sure that this texture load is an add of the base
208 		 * address of the UBO to a clamped offset within the UBO.
209 		 */
210 		if (is_mul ||
211 		    QPU_GET_FIELD(inst, QPU_OP_ADD) != QPU_A_ADD) {
212 			DRM_DEBUG("direct TMU load wasn't an add\n");
213 			return false;
214 		}
215 
216 		/* We assert that the clamped address is the first
217 		 * argument, and the UBO base address is the second argument.
218 		 * This is arbitrary, but simpler than supporting flipping the
219 		 * two either way.
220 		 */
221 		clamp_reg = raddr_add_a_to_live_reg_index(inst);
222 		if (clamp_reg == ~0) {
223 			DRM_DEBUG("direct TMU load wasn't clamped\n");
224 			return false;
225 		}
226 
227 		clamp_offset = validation_state->live_min_clamp_offsets[clamp_reg];
228 		if (clamp_offset == ~0) {
229 			DRM_DEBUG("direct TMU load wasn't clamped\n");
230 			return false;
231 		}
232 
233 		/* Store the clamp value's offset in p1 (see reloc_tex() in
234 		 * vc4_validate.c).
235 		 */
236 		validation_state->tmu_setup[tmu].p_offset[1] =
237 			clamp_offset;
238 
239 		if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
240 		    !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF)) {
241 			DRM_DEBUG("direct TMU load didn't add to a uniform\n");
242 			return false;
243 		}
244 
245 		validation_state->tmu_setup[tmu].is_direct = true;
246 	} else {
247 		if (raddr_a == QPU_R_UNIF || (sig != QPU_SIG_SMALL_IMM &&
248 					      raddr_b == QPU_R_UNIF)) {
249 			DRM_DEBUG("uniform read in the same instruction as "
250 				  "texture setup.\n");
251 			return false;
252 		}
253 	}
254 
255 	if (validation_state->tmu_write_count[tmu] >= 4) {
256 		DRM_DEBUG("TMU%d got too many parameters before dispatch\n",
257 			  tmu);
258 		return false;
259 	}
260 	validation_state->tmu_setup[tmu].p_offset[validation_state->tmu_write_count[tmu]] =
261 		validated_shader->uniforms_size;
262 	validation_state->tmu_write_count[tmu]++;
263 	/* Since direct uses a RADDR uniform reference, it will get counted in
264 	 * check_instruction_reads()
265 	 */
266 	if (!is_direct) {
267 		if (validation_state->needs_uniform_address_update) {
268 			DRM_DEBUG("Texturing with undefined uniform address\n");
269 			return false;
270 		}
271 
272 		validated_shader->uniforms_size += 4;
273 	}
274 
275 	if (submit) {
276 		if (!record_texture_sample(validated_shader,
277 					   validation_state, tmu)) {
278 			return false;
279 		}
280 
281 		validation_state->tmu_write_count[tmu] = 0;
282 	}
283 
284 	return true;
285 }
286 
require_uniform_address_uniform(struct vc4_validated_shader_info * validated_shader)287 static bool require_uniform_address_uniform(struct vc4_validated_shader_info *validated_shader)
288 {
289 	uint32_t o = validated_shader->num_uniform_addr_offsets;
290 	uint32_t num_uniforms = validated_shader->uniforms_size / 4;
291 
292 	validated_shader->uniform_addr_offsets =
293 		krealloc(validated_shader->uniform_addr_offsets,
294 			 (o + 1) *
295 			 sizeof(*validated_shader->uniform_addr_offsets),
296 			 GFP_KERNEL);
297 	if (!validated_shader->uniform_addr_offsets)
298 		return false;
299 
300 	validated_shader->uniform_addr_offsets[o] = num_uniforms;
301 	validated_shader->num_uniform_addr_offsets++;
302 
303 	return true;
304 }
305 
306 static bool
validate_uniform_address_write(struct vc4_validated_shader_info * validated_shader,struct vc4_shader_validation_state * validation_state,bool is_mul)307 validate_uniform_address_write(struct vc4_validated_shader_info *validated_shader,
308 			       struct vc4_shader_validation_state *validation_state,
309 			       bool is_mul)
310 {
311 	uint64_t inst = validation_state->shader[validation_state->ip];
312 	u32 add_b = QPU_GET_FIELD(inst, QPU_ADD_B);
313 	u32 raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
314 	u32 raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
315 	u32 add_lri = raddr_add_a_to_live_reg_index(inst);
316 	/* We want our reset to be pointing at whatever uniform follows the
317 	 * uniforms base address.
318 	 */
319 	u32 expected_offset = validated_shader->uniforms_size + 4;
320 
321 	/* We only support absolute uniform address changes, and we
322 	 * require that they be in the current basic block before any
323 	 * of its uniform reads.
324 	 *
325 	 * One could potentially emit more efficient QPU code, by
326 	 * noticing that (say) an if statement does uniform control
327 	 * flow for all threads and that the if reads the same number
328 	 * of uniforms on each side.  However, this scheme is easy to
329 	 * validate so it's all we allow for now.
330 	 */
331 	switch (QPU_GET_FIELD(inst, QPU_SIG)) {
332 	case QPU_SIG_NONE:
333 	case QPU_SIG_SCOREBOARD_UNLOCK:
334 	case QPU_SIG_COLOR_LOAD:
335 	case QPU_SIG_LOAD_TMU0:
336 	case QPU_SIG_LOAD_TMU1:
337 		break;
338 	default:
339 		DRM_DEBUG("uniforms address change must be "
340 			  "normal math\n");
341 		return false;
342 	}
343 
344 	if (is_mul || QPU_GET_FIELD(inst, QPU_OP_ADD) != QPU_A_ADD) {
345 		DRM_DEBUG("Uniform address reset must be an ADD.\n");
346 		return false;
347 	}
348 
349 	if (QPU_GET_FIELD(inst, QPU_COND_ADD) != QPU_COND_ALWAYS) {
350 		DRM_DEBUG("Uniform address reset must be unconditional.\n");
351 		return false;
352 	}
353 
354 	if (QPU_GET_FIELD(inst, QPU_PACK) != QPU_PACK_A_NOP &&
355 	    !(inst & QPU_PM)) {
356 		DRM_DEBUG("No packing allowed on uniforms reset\n");
357 		return false;
358 	}
359 
360 	if (add_lri == -1) {
361 		DRM_DEBUG("First argument of uniform address write must be "
362 			  "an immediate value.\n");
363 		return false;
364 	}
365 
366 	if (validation_state->live_immediates[add_lri] != expected_offset) {
367 		DRM_DEBUG("Resetting uniforms with offset %db instead of %db\n",
368 			  validation_state->live_immediates[add_lri],
369 			  expected_offset);
370 		return false;
371 	}
372 
373 	if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
374 	    !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF)) {
375 		DRM_DEBUG("Second argument of uniform address write must be "
376 			  "a uniform.\n");
377 		return false;
378 	}
379 
380 	validation_state->needs_uniform_address_update = false;
381 	validation_state->needs_uniform_address_for_loop = false;
382 	return require_uniform_address_uniform(validated_shader);
383 }
384 
385 static bool
check_reg_write(struct vc4_validated_shader_info * validated_shader,struct vc4_shader_validation_state * validation_state,bool is_mul)386 check_reg_write(struct vc4_validated_shader_info *validated_shader,
387 		struct vc4_shader_validation_state *validation_state,
388 		bool is_mul)
389 {
390 	uint64_t inst = validation_state->shader[validation_state->ip];
391 	uint32_t waddr = (is_mul ?
392 			  QPU_GET_FIELD(inst, QPU_WADDR_MUL) :
393 			  QPU_GET_FIELD(inst, QPU_WADDR_ADD));
394 	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
395 	bool ws = inst & QPU_WS;
396 	bool is_b = is_mul ^ ws;
397 	u32 lri = waddr_to_live_reg_index(waddr, is_b);
398 
399 	if (lri != -1) {
400 		uint32_t cond_add = QPU_GET_FIELD(inst, QPU_COND_ADD);
401 		uint32_t cond_mul = QPU_GET_FIELD(inst, QPU_COND_MUL);
402 
403 		if (sig == QPU_SIG_LOAD_IMM &&
404 		    QPU_GET_FIELD(inst, QPU_PACK) == QPU_PACK_A_NOP &&
405 		    ((is_mul && cond_mul == QPU_COND_ALWAYS) ||
406 		     (!is_mul && cond_add == QPU_COND_ALWAYS))) {
407 			validation_state->live_immediates[lri] =
408 				QPU_GET_FIELD(inst, QPU_LOAD_IMM);
409 		} else {
410 			validation_state->live_immediates[lri] = ~0;
411 		}
412 
413 		if (live_reg_is_upper_half(lri))
414 			validation_state->all_registers_used = true;
415 	}
416 
417 	switch (waddr) {
418 	case QPU_W_UNIFORMS_ADDRESS:
419 		if (is_b) {
420 			DRM_DEBUG("relative uniforms address change "
421 				  "unsupported\n");
422 			return false;
423 		}
424 
425 		return validate_uniform_address_write(validated_shader,
426 						      validation_state,
427 						      is_mul);
428 
429 	case QPU_W_TLB_COLOR_MS:
430 	case QPU_W_TLB_COLOR_ALL:
431 	case QPU_W_TLB_Z:
432 		/* These only interact with the tile buffer, not main memory,
433 		 * so they're safe.
434 		 */
435 		return true;
436 
437 	case QPU_W_TMU0_S:
438 	case QPU_W_TMU0_T:
439 	case QPU_W_TMU0_R:
440 	case QPU_W_TMU0_B:
441 	case QPU_W_TMU1_S:
442 	case QPU_W_TMU1_T:
443 	case QPU_W_TMU1_R:
444 	case QPU_W_TMU1_B:
445 		return check_tmu_write(validated_shader, validation_state,
446 				       is_mul);
447 
448 	case QPU_W_HOST_INT:
449 	case QPU_W_TMU_NOSWAP:
450 	case QPU_W_TLB_ALPHA_MASK:
451 	case QPU_W_MUTEX_RELEASE:
452 		/* XXX: I haven't thought about these, so don't support them
453 		 * for now.
454 		 */
455 		DRM_DEBUG("Unsupported waddr %d\n", waddr);
456 		return false;
457 
458 	case QPU_W_VPM_ADDR:
459 		DRM_DEBUG("General VPM DMA unsupported\n");
460 		return false;
461 
462 	case QPU_W_VPM:
463 	case QPU_W_VPMVCD_SETUP:
464 		/* We allow VPM setup in general, even including VPM DMA
465 		 * configuration setup, because the (unsafe) DMA can only be
466 		 * triggered by QPU_W_VPM_ADDR writes.
467 		 */
468 		return true;
469 
470 	case QPU_W_TLB_STENCIL_SETUP:
471 		return true;
472 	}
473 
474 	return true;
475 }
476 
477 static void
track_live_clamps(struct vc4_validated_shader_info * validated_shader,struct vc4_shader_validation_state * validation_state)478 track_live_clamps(struct vc4_validated_shader_info *validated_shader,
479 		  struct vc4_shader_validation_state *validation_state)
480 {
481 	uint64_t inst = validation_state->shader[validation_state->ip];
482 	uint32_t op_add = QPU_GET_FIELD(inst, QPU_OP_ADD);
483 	uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
484 	uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
485 	uint32_t cond_add = QPU_GET_FIELD(inst, QPU_COND_ADD);
486 	uint32_t add_a = QPU_GET_FIELD(inst, QPU_ADD_A);
487 	uint32_t add_b = QPU_GET_FIELD(inst, QPU_ADD_B);
488 	uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
489 	uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
490 	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
491 	bool ws = inst & QPU_WS;
492 	uint32_t lri_add_a, lri_add, lri_mul;
493 	bool add_a_is_min_0;
494 
495 	/* Check whether OP_ADD's A argumennt comes from a live MAX(x, 0),
496 	 * before we clear previous live state.
497 	 */
498 	lri_add_a = raddr_add_a_to_live_reg_index(inst);
499 	add_a_is_min_0 = (lri_add_a != ~0 &&
500 			  validation_state->live_max_clamp_regs[lri_add_a]);
501 
502 	/* Clear live state for registers written by our instruction. */
503 	lri_add = waddr_to_live_reg_index(waddr_add, ws);
504 	lri_mul = waddr_to_live_reg_index(waddr_mul, !ws);
505 	if (lri_mul != ~0) {
506 		validation_state->live_max_clamp_regs[lri_mul] = false;
507 		validation_state->live_min_clamp_offsets[lri_mul] = ~0;
508 	}
509 	if (lri_add != ~0) {
510 		validation_state->live_max_clamp_regs[lri_add] = false;
511 		validation_state->live_min_clamp_offsets[lri_add] = ~0;
512 	} else {
513 		/* Nothing further to do for live tracking, since only ADDs
514 		 * generate new live clamp registers.
515 		 */
516 		return;
517 	}
518 
519 	/* Now, handle remaining live clamp tracking for the ADD operation. */
520 
521 	if (cond_add != QPU_COND_ALWAYS)
522 		return;
523 
524 	if (op_add == QPU_A_MAX) {
525 		/* Track live clamps of a value to a minimum of 0 (in either
526 		 * arg).
527 		 */
528 		if (sig != QPU_SIG_SMALL_IMM || raddr_b != 0 ||
529 		    (add_a != QPU_MUX_B && add_b != QPU_MUX_B)) {
530 			return;
531 		}
532 
533 		validation_state->live_max_clamp_regs[lri_add] = true;
534 	} else if (op_add == QPU_A_MIN) {
535 		/* Track live clamps of a value clamped to a minimum of 0 and
536 		 * a maximum of some uniform's offset.
537 		 */
538 		if (!add_a_is_min_0)
539 			return;
540 
541 		if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
542 		    !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF &&
543 		      sig != QPU_SIG_SMALL_IMM)) {
544 			return;
545 		}
546 
547 		validation_state->live_min_clamp_offsets[lri_add] =
548 			validated_shader->uniforms_size;
549 	}
550 }
551 
552 static bool
check_instruction_writes(struct vc4_validated_shader_info * validated_shader,struct vc4_shader_validation_state * validation_state)553 check_instruction_writes(struct vc4_validated_shader_info *validated_shader,
554 			 struct vc4_shader_validation_state *validation_state)
555 {
556 	uint64_t inst = validation_state->shader[validation_state->ip];
557 	uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
558 	uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
559 	bool ok;
560 
561 	if (is_tmu_write(waddr_add) && is_tmu_write(waddr_mul)) {
562 		DRM_DEBUG("ADD and MUL both set up textures\n");
563 		return false;
564 	}
565 
566 	ok = (check_reg_write(validated_shader, validation_state, false) &&
567 	      check_reg_write(validated_shader, validation_state, true));
568 
569 	track_live_clamps(validated_shader, validation_state);
570 
571 	return ok;
572 }
573 
574 static bool
check_branch(uint64_t inst,struct vc4_validated_shader_info * validated_shader,struct vc4_shader_validation_state * validation_state,int ip)575 check_branch(uint64_t inst,
576 	     struct vc4_validated_shader_info *validated_shader,
577 	     struct vc4_shader_validation_state *validation_state,
578 	     int ip)
579 {
580 	int32_t branch_imm = QPU_GET_FIELD(inst, QPU_BRANCH_TARGET);
581 	uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
582 	uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
583 
584 	if ((int)branch_imm < 0)
585 		validation_state->needs_uniform_address_for_loop = true;
586 
587 	/* We don't want to have to worry about validation of this, and
588 	 * there's no need for it.
589 	 */
590 	if (waddr_add != QPU_W_NOP || waddr_mul != QPU_W_NOP) {
591 		DRM_DEBUG("branch instruction at %d wrote a register.\n",
592 			  validation_state->ip);
593 		return false;
594 	}
595 
596 	return true;
597 }
598 
599 static bool
check_instruction_reads(struct vc4_validated_shader_info * validated_shader,struct vc4_shader_validation_state * validation_state)600 check_instruction_reads(struct vc4_validated_shader_info *validated_shader,
601 			struct vc4_shader_validation_state *validation_state)
602 {
603 	uint64_t inst = validation_state->shader[validation_state->ip];
604 	uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
605 	uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
606 	uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
607 
608 	if (raddr_a == QPU_R_UNIF ||
609 	    (raddr_b == QPU_R_UNIF && sig != QPU_SIG_SMALL_IMM)) {
610 		/* This can't overflow the uint32_t, because we're reading 8
611 		 * bytes of instruction to increment by 4 here, so we'd
612 		 * already be OOM.
613 		 */
614 		validated_shader->uniforms_size += 4;
615 
616 		if (validation_state->needs_uniform_address_update) {
617 			DRM_DEBUG("Uniform read with undefined uniform "
618 				  "address\n");
619 			return false;
620 		}
621 	}
622 
623 	if ((raddr_a >= 16 && raddr_a < 32) ||
624 	    (raddr_b >= 16 && raddr_b < 32 && sig != QPU_SIG_SMALL_IMM)) {
625 		validation_state->all_registers_used = true;
626 	}
627 
628 	return true;
629 }
630 
631 /* Make sure that all branches are absolute and point within the shader, and
632  * note their targets for later.
633  */
634 static bool
vc4_validate_branches(struct vc4_shader_validation_state * validation_state)635 vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
636 {
637 	uint32_t max_branch_target = 0;
638 	int ip;
639 	int last_branch = -2;
640 
641 	for (ip = 0; ip < validation_state->max_ip; ip++) {
642 		uint64_t inst = validation_state->shader[ip];
643 		int32_t branch_imm = QPU_GET_FIELD(inst, QPU_BRANCH_TARGET);
644 		uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
645 		uint32_t after_delay_ip = ip + 4;
646 		uint32_t branch_target_ip;
647 
648 		if (sig == QPU_SIG_PROG_END) {
649 			/* There are two delay slots after program end is
650 			 * signaled that are still executed, then we're
651 			 * finished.  validation_state->max_ip is the
652 			 * instruction after the last valid instruction in the
653 			 * program.
654 			 */
655 			validation_state->max_ip = ip + 3;
656 			continue;
657 		}
658 
659 		if (sig != QPU_SIG_BRANCH)
660 			continue;
661 
662 		if (ip - last_branch < 4) {
663 			DRM_DEBUG("Branch at %d during delay slots\n", ip);
664 			return false;
665 		}
666 		last_branch = ip;
667 
668 		if (inst & QPU_BRANCH_REG) {
669 			DRM_DEBUG("branching from register relative "
670 				  "not supported\n");
671 			return false;
672 		}
673 
674 		if (!(inst & QPU_BRANCH_REL)) {
675 			DRM_DEBUG("relative branching required\n");
676 			return false;
677 		}
678 
679 		/* The actual branch target is the instruction after the delay
680 		 * slots, plus whatever byte offset is in the low 32 bits of
681 		 * the instruction.  Make sure we're not branching beyond the
682 		 * end of the shader object.
683 		 */
684 		if (branch_imm % sizeof(inst) != 0) {
685 			DRM_DEBUG("branch target not aligned\n");
686 			return false;
687 		}
688 
689 		branch_target_ip = after_delay_ip + (branch_imm >> 3);
690 		if (branch_target_ip >= validation_state->max_ip) {
691 			DRM_DEBUG("Branch at %d outside of shader (ip %d/%d)\n",
692 				  ip, branch_target_ip,
693 				  validation_state->max_ip);
694 			return false;
695 		}
696 		set_bit(branch_target_ip, validation_state->branch_targets);
697 
698 		/* Make sure that the non-branching path is also not outside
699 		 * the shader.
700 		 */
701 		if (after_delay_ip >= validation_state->max_ip) {
702 			DRM_DEBUG("Branch at %d continues past shader end "
703 				  "(%d/%d)\n",
704 				  ip, after_delay_ip, validation_state->max_ip);
705 			return false;
706 		}
707 		set_bit(after_delay_ip, validation_state->branch_targets);
708 		max_branch_target = max(max_branch_target, after_delay_ip);
709 	}
710 
711 	if (max_branch_target > validation_state->max_ip - 3) {
712 		DRM_DEBUG("Branch landed after QPU_SIG_PROG_END");
713 		return false;
714 	}
715 
716 	return true;
717 }
718 
719 /* Resets any known state for the shader, used when we may be branched to from
720  * multiple locations in the program (or at shader start).
721  */
722 static void
reset_validation_state(struct vc4_shader_validation_state * validation_state)723 reset_validation_state(struct vc4_shader_validation_state *validation_state)
724 {
725 	int i;
726 
727 	for (i = 0; i < 8; i++)
728 		validation_state->tmu_setup[i / 4].p_offset[i % 4] = ~0;
729 
730 	for (i = 0; i < LIVE_REG_COUNT; i++) {
731 		validation_state->live_min_clamp_offsets[i] = ~0;
732 		validation_state->live_max_clamp_regs[i] = false;
733 		validation_state->live_immediates[i] = ~0;
734 	}
735 }
736 
737 static bool
texturing_in_progress(struct vc4_shader_validation_state * validation_state)738 texturing_in_progress(struct vc4_shader_validation_state *validation_state)
739 {
740 	return (validation_state->tmu_write_count[0] != 0 ||
741 		validation_state->tmu_write_count[1] != 0);
742 }
743 
744 static bool
vc4_handle_branch_target(struct vc4_shader_validation_state * validation_state)745 vc4_handle_branch_target(struct vc4_shader_validation_state *validation_state)
746 {
747 	uint32_t ip = validation_state->ip;
748 
749 	if (!test_bit(ip, validation_state->branch_targets))
750 		return true;
751 
752 	if (texturing_in_progress(validation_state)) {
753 		DRM_DEBUG("Branch target landed during TMU setup\n");
754 		return false;
755 	}
756 
757 	/* Reset our live values tracking, since this instruction may have
758 	 * multiple predecessors.
759 	 *
760 	 * One could potentially do analysis to determine that, for
761 	 * example, all predecessors have a live max clamp in the same
762 	 * register, but we don't bother with that.
763 	 */
764 	reset_validation_state(validation_state);
765 
766 	/* Since we've entered a basic block from potentially multiple
767 	 * predecessors, we need the uniforms address to be updated before any
768 	 * unforms are read.  We require that after any branch point, the next
769 	 * uniform to be loaded is a uniform address offset.  That uniform's
770 	 * offset will be marked by the uniform address register write
771 	 * validation, or a one-off the end-of-program check.
772 	 */
773 	validation_state->needs_uniform_address_update = true;
774 
775 	return true;
776 }
777 
778 struct vc4_validated_shader_info *
vc4_validate_shader(struct drm_gem_dma_object * shader_obj)779 vc4_validate_shader(struct drm_gem_dma_object *shader_obj)
780 {
781 	struct vc4_dev *vc4 = to_vc4_dev(shader_obj->base.dev);
782 	bool found_shader_end = false;
783 	int shader_end_ip = 0;
784 	uint32_t last_thread_switch_ip = -3;
785 	uint32_t ip;
786 	struct vc4_validated_shader_info *validated_shader = NULL;
787 	struct vc4_shader_validation_state validation_state;
788 
789 	if (WARN_ON_ONCE(vc4->is_vc5))
790 		return NULL;
791 
792 	memset(&validation_state, 0, sizeof(validation_state));
793 	validation_state.shader = shader_obj->vaddr;
794 	validation_state.max_ip = shader_obj->base.size / sizeof(uint64_t);
795 
796 	reset_validation_state(&validation_state);
797 
798 	validation_state.branch_targets =
799 		kcalloc(BITS_TO_LONGS(validation_state.max_ip),
800 			sizeof(unsigned long), GFP_KERNEL);
801 	if (!validation_state.branch_targets)
802 		goto fail;
803 
804 	validated_shader = kcalloc(1, sizeof(*validated_shader), GFP_KERNEL);
805 	if (!validated_shader)
806 		goto fail;
807 
808 	if (!vc4_validate_branches(&validation_state))
809 		goto fail;
810 
811 	for (ip = 0; ip < validation_state.max_ip; ip++) {
812 		uint64_t inst = validation_state.shader[ip];
813 		uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
814 
815 		validation_state.ip = ip;
816 
817 		if (!vc4_handle_branch_target(&validation_state))
818 			goto fail;
819 
820 		if (ip == last_thread_switch_ip + 3) {
821 			/* Reset r0-r3 live clamp data */
822 			int i;
823 
824 			for (i = 64; i < LIVE_REG_COUNT; i++) {
825 				validation_state.live_min_clamp_offsets[i] = ~0;
826 				validation_state.live_max_clamp_regs[i] = false;
827 				validation_state.live_immediates[i] = ~0;
828 			}
829 		}
830 
831 		switch (sig) {
832 		case QPU_SIG_NONE:
833 		case QPU_SIG_WAIT_FOR_SCOREBOARD:
834 		case QPU_SIG_SCOREBOARD_UNLOCK:
835 		case QPU_SIG_COLOR_LOAD:
836 		case QPU_SIG_LOAD_TMU0:
837 		case QPU_SIG_LOAD_TMU1:
838 		case QPU_SIG_PROG_END:
839 		case QPU_SIG_SMALL_IMM:
840 		case QPU_SIG_THREAD_SWITCH:
841 		case QPU_SIG_LAST_THREAD_SWITCH:
842 			if (!check_instruction_writes(validated_shader,
843 						      &validation_state)) {
844 				DRM_DEBUG("Bad write at ip %d\n", ip);
845 				goto fail;
846 			}
847 
848 			if (!check_instruction_reads(validated_shader,
849 						     &validation_state))
850 				goto fail;
851 
852 			if (sig == QPU_SIG_PROG_END) {
853 				found_shader_end = true;
854 				shader_end_ip = ip;
855 			}
856 
857 			if (sig == QPU_SIG_THREAD_SWITCH ||
858 			    sig == QPU_SIG_LAST_THREAD_SWITCH) {
859 				validated_shader->is_threaded = true;
860 
861 				if (ip < last_thread_switch_ip + 3) {
862 					DRM_DEBUG("Thread switch too soon after "
863 						  "last switch at ip %d\n", ip);
864 					goto fail;
865 				}
866 				last_thread_switch_ip = ip;
867 			}
868 
869 			break;
870 
871 		case QPU_SIG_LOAD_IMM:
872 			if (!check_instruction_writes(validated_shader,
873 						      &validation_state)) {
874 				DRM_DEBUG("Bad LOAD_IMM write at ip %d\n", ip);
875 				goto fail;
876 			}
877 			break;
878 
879 		case QPU_SIG_BRANCH:
880 			if (!check_branch(inst, validated_shader,
881 					  &validation_state, ip))
882 				goto fail;
883 
884 			if (ip < last_thread_switch_ip + 3) {
885 				DRM_DEBUG("Branch in thread switch at ip %d",
886 					  ip);
887 				goto fail;
888 			}
889 
890 			break;
891 		default:
892 			DRM_DEBUG("Unsupported QPU signal %d at "
893 				  "instruction %d\n", sig, ip);
894 			goto fail;
895 		}
896 
897 		/* There are two delay slots after program end is signaled
898 		 * that are still executed, then we're finished.
899 		 */
900 		if (found_shader_end && ip == shader_end_ip + 2)
901 			break;
902 	}
903 
904 	if (ip == validation_state.max_ip) {
905 		DRM_DEBUG("shader failed to terminate before "
906 			  "shader BO end at %zd\n",
907 			  shader_obj->base.size);
908 		goto fail;
909 	}
910 
911 	/* Might corrupt other thread */
912 	if (validated_shader->is_threaded &&
913 	    validation_state.all_registers_used) {
914 		DRM_DEBUG("Shader uses threading, but uses the upper "
915 			  "half of the registers, too\n");
916 		goto fail;
917 	}
918 
919 	/* If we did a backwards branch and we haven't emitted a uniforms
920 	 * reset since then, we still need the uniforms stream to have the
921 	 * uniforms address available so that the backwards branch can do its
922 	 * uniforms reset.
923 	 *
924 	 * We could potentially prove that the backwards branch doesn't
925 	 * contain any uses of uniforms until program exit, but that doesn't
926 	 * seem to be worth the trouble.
927 	 */
928 	if (validation_state.needs_uniform_address_for_loop) {
929 		if (!require_uniform_address_uniform(validated_shader))
930 			goto fail;
931 		validated_shader->uniforms_size += 4;
932 	}
933 
934 	/* Again, no chance of integer overflow here because the worst case
935 	 * scenario is 8 bytes of uniforms plus handles per 8-byte
936 	 * instruction.
937 	 */
938 	validated_shader->uniforms_src_size =
939 		(validated_shader->uniforms_size +
940 		 4 * validated_shader->num_texture_samples);
941 
942 	kfree(validation_state.branch_targets);
943 
944 	return validated_shader;
945 
946 fail:
947 	kfree(validation_state.branch_targets);
948 	if (validated_shader) {
949 		kfree(validated_shader->uniform_addr_offsets);
950 		kfree(validated_shader->texture_samples);
951 		kfree(validated_shader);
952 	}
953 	return NULL;
954 }
955