1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2021 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7 
8 #include <linux/vmalloc.h>
9 #include <uapi/drm/habanalabs_accel.h>
10 #include "habanalabs.h"
11 
12 /**
13  * hl_format_as_binary - helper function, format an integer as binary
14  *                       using supplied scratch buffer
15  * @buf: the buffer to use
16  * @buf_len: buffer capacity
17  * @n: number to format
18  *
19  * Returns pointer to buffer
20  */
hl_format_as_binary(char * buf,size_t buf_len,u32 n)21 char *hl_format_as_binary(char *buf, size_t buf_len, u32 n)
22 {
23 	int i;
24 	u32 bit;
25 	bool leading0 = true;
26 	char *wrptr = buf;
27 
28 	if (buf_len > 0 && buf_len < 3) {
29 		*wrptr = '\0';
30 		return buf;
31 	}
32 
33 	wrptr[0] = '0';
34 	wrptr[1] = 'b';
35 	wrptr += 2;
36 	/* Remove 3 characters from length for '0b' and '\0' termination */
37 	buf_len -= 3;
38 
39 	for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) {
40 		/* Writing bit calculation in one line would cause a false
41 		 * positive static code analysis error, so splitting.
42 		 */
43 		bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1));
44 		bit = !!bit;
45 		leading0 &= !bit;
46 		if (!leading0) {
47 			*wrptr = '0' + bit;
48 			++wrptr;
49 		}
50 	}
51 
52 	*wrptr = '\0';
53 
54 	return buf;
55 }
56 
57 /**
58  * resize_to_fit - helper function, resize buffer to fit given amount of data
59  * @buf: destination buffer double pointer
60  * @size: pointer to the size container
61  * @desired_size: size the buffer must contain
62  *
63  * Returns 0 on success or error code on failure.
64  * On success, the size of buffer is at least desired_size. Buffer is allocated
65  * via vmalloc and must be freed with vfree.
66  */
resize_to_fit(char ** buf,size_t * size,size_t desired_size)67 static int resize_to_fit(char **buf, size_t *size, size_t desired_size)
68 {
69 	char *resized_buf;
70 	size_t new_size;
71 
72 	if (*size >= desired_size)
73 		return 0;
74 
75 	/* Not enough space to print all, have to resize */
76 	new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE));
77 	resized_buf = vmalloc(new_size);
78 	if (!resized_buf)
79 		return -ENOMEM;
80 	memcpy(resized_buf, *buf, *size);
81 	vfree(*buf);
82 	*buf = resized_buf;
83 	*size = new_size;
84 
85 	return 1;
86 }
87 
88 /**
89  * hl_snprintf_resize() - print formatted data to buffer, resize as needed
90  * @buf: buffer double pointer, to be written to and resized, must be either
91  *       NULL or allocated with vmalloc.
92  * @size: current size of the buffer
93  * @offset: current offset to write to
94  * @format: format of the data
95  *
96  * This function will write formatted data into the buffer. If buffer is not
97  * large enough, it will be resized using vmalloc. Size may be modified if the
98  * buffer was resized, offset will be advanced by the number of bytes written
99  * not including the terminating character
100  *
101  * Returns 0 on success or error code on failure
102  *
103  * Note that the buffer has to be manually released using vfree.
104  */
hl_snprintf_resize(char ** buf,size_t * size,size_t * offset,const char * format,...)105 int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,
106 			   const char *format, ...)
107 {
108 	va_list args;
109 	size_t length;
110 	int rc;
111 
112 	if (*buf == NULL && (*size != 0 || *offset != 0))
113 		return -EINVAL;
114 
115 	va_start(args, format);
116 	length = vsnprintf(*buf + *offset, *size - *offset, format, args);
117 	va_end(args);
118 
119 	rc = resize_to_fit(buf, size, *offset + length + 1);
120 	if (rc < 0)
121 		return rc;
122 	else if (rc > 0) {
123 		/* Resize was needed, write again */
124 		va_start(args, format);
125 		length = vsnprintf(*buf + *offset, *size - *offset, format,
126 				   args);
127 		va_end(args);
128 	}
129 
130 	*offset += length;
131 
132 	return 0;
133 }
134 
135 /**
136  * hl_sync_engine_to_string - convert engine type enum to string literal
137  * @engine_type: engine type (TPC/MME/DMA)
138  *
139  * Return the resolved string literal
140  */
hl_sync_engine_to_string(enum hl_sync_engine_type engine_type)141 const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type)
142 {
143 	switch (engine_type) {
144 	case ENGINE_DMA:
145 		return "DMA";
146 	case ENGINE_MME:
147 		return "MME";
148 	case ENGINE_TPC:
149 		return "TPC";
150 	}
151 	return "Invalid Engine Type";
152 }
153 
154 /**
155  * hl_print_resize_sync_engine - helper function, format engine name and ID
156  * using hl_snprintf_resize
157  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
158  * @size: pointer to the size container
159  * @offset: pointer to the offset container
160  * @engine_type: engine type (TPC/MME/DMA)
161  * @engine_id: engine numerical id
162  *
163  * Returns 0 on success or error code on failure
164  */
hl_print_resize_sync_engine(char ** buf,size_t * size,size_t * offset,enum hl_sync_engine_type engine_type,u32 engine_id)165 static int hl_print_resize_sync_engine(char **buf, size_t *size, size_t *offset,
166 				enum hl_sync_engine_type engine_type,
167 				u32 engine_id)
168 {
169 	return hl_snprintf_resize(buf, size, offset, "%s%u",
170 			hl_sync_engine_to_string(engine_type), engine_id);
171 }
172 
173 /**
174  * hl_state_dump_get_sync_name - transform sync object id to name if available
175  * @hdev: pointer to the device
176  * @sync_id: sync object id
177  *
178  * Returns a name literal or NULL if not resolved.
179  * Note: returning NULL shall not be considered as a failure, as not all
180  * sync objects are named.
181  */
hl_state_dump_get_sync_name(struct hl_device * hdev,u32 sync_id)182 const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id)
183 {
184 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
185 	struct hl_hw_obj_name_entry *entry;
186 
187 	hash_for_each_possible(sds->so_id_to_str_tb, entry,
188 				node, sync_id)
189 		if (sync_id == entry->id)
190 			return entry->name;
191 
192 	return NULL;
193 }
194 
195 /**
196  * hl_state_dump_get_monitor_name - transform monitor object dump to monitor
197  * name if available
198  * @hdev: pointer to the device
199  * @mon: monitor state dump
200  *
201  * Returns a name literal or NULL if not resolved.
202  * Note: returning NULL shall not be considered as a failure, as not all
203  * monitors are named.
204  */
hl_state_dump_get_monitor_name(struct hl_device * hdev,struct hl_mon_state_dump * mon)205 const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,
206 					struct hl_mon_state_dump *mon)
207 {
208 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
209 	struct hl_hw_obj_name_entry *entry;
210 
211 	hash_for_each_possible(sds->monitor_id_to_str_tb,
212 				entry, node, mon->id)
213 		if (mon->id == entry->id)
214 			return entry->name;
215 
216 	return NULL;
217 }
218 
219 /**
220  * hl_state_dump_free_sync_to_engine_map - free sync object to engine map
221  * @map: sync object to engine map
222  *
223  * Note: generic free implementation, the allocation is implemented per ASIC.
224  */
hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map * map)225 void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map)
226 {
227 	struct hl_sync_to_engine_map_entry *entry;
228 	struct hlist_node *tmp_node;
229 	int i;
230 
231 	hash_for_each_safe(map->tb, i, tmp_node, entry, node) {
232 		hash_del(&entry->node);
233 		kfree(entry);
234 	}
235 }
236 
237 /**
238  * hl_state_dump_get_sync_to_engine - transform sync_id to
239  * hl_sync_to_engine_map_entry if available for current id
240  * @map: sync object to engine map
241  * @sync_id: sync object id
242  *
243  * Returns the translation entry if found or NULL if not.
244  * Note, returned NULL shall not be considered as a failure as the map
245  * does not cover all possible, it is a best effort sync ids.
246  */
247 static struct hl_sync_to_engine_map_entry *
hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map * map,u32 sync_id)248 hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id)
249 {
250 	struct hl_sync_to_engine_map_entry *entry;
251 
252 	hash_for_each_possible(map->tb, entry, node, sync_id)
253 		if (entry->sync_id == sync_id)
254 			return entry;
255 	return NULL;
256 }
257 
258 /**
259  * hl_state_dump_read_sync_objects - read sync objects array
260  * @hdev: pointer to the device
261  * @index: sync manager block index starting with E_N
262  *
263  * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure
264  */
hl_state_dump_read_sync_objects(struct hl_device * hdev,u32 index)265 static u32 *hl_state_dump_read_sync_objects(struct hl_device *hdev, u32 index)
266 {
267 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
268 	u32 *sync_objects;
269 	s64 base_addr; /* Base addr can be negative */
270 	int i;
271 
272 	base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
273 			sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
274 
275 	sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32));
276 	if (!sync_objects)
277 		return NULL;
278 
279 	for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i)
280 		sync_objects[i] = RREG32(base_addr + i * sizeof(u32));
281 
282 	return sync_objects;
283 }
284 
285 /**
286  * hl_state_dump_free_sync_objects - free sync objects array allocated by
287  * hl_state_dump_read_sync_objects
288  * @sync_objects: sync objects array
289  */
hl_state_dump_free_sync_objects(u32 * sync_objects)290 static void hl_state_dump_free_sync_objects(u32 *sync_objects)
291 {
292 	vfree(sync_objects);
293 }
294 
295 
296 /**
297  * hl_state_dump_print_syncs_single_block - print active sync objects on a
298  * single block
299  * @hdev: pointer to the device
300  * @index: sync manager block index starting with E_N
301  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
302  * @size: pointer to the size container
303  * @offset: pointer to the offset container
304  * @map: sync engines names map
305  *
306  * Returns 0 on success or error code on failure
307  */
308 static int
hl_state_dump_print_syncs_single_block(struct hl_device * hdev,u32 index,char ** buf,size_t * size,size_t * offset,struct hl_sync_to_engine_map * map)309 hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index,
310 				char **buf, size_t *size, size_t *offset,
311 				struct hl_sync_to_engine_map *map)
312 {
313 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
314 	const char *sync_name;
315 	u32 *sync_objects = NULL;
316 	int rc = 0, i;
317 
318 	if (sds->sync_namager_names) {
319 		rc = hl_snprintf_resize(
320 			buf, size, offset, "%s\n",
321 			sds->sync_namager_names[index]);
322 		if (rc)
323 			goto out;
324 	}
325 
326 	sync_objects = hl_state_dump_read_sync_objects(hdev, index);
327 	if (!sync_objects) {
328 		rc = -ENOMEM;
329 		goto out;
330 	}
331 
332 	for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) {
333 		struct hl_sync_to_engine_map_entry *entry;
334 		u64 sync_object_addr;
335 
336 		if (!sync_objects[i])
337 			continue;
338 
339 		sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +
340 				sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index +
341 				i * sizeof(u32);
342 
343 		rc = hl_snprintf_resize(buf, size, offset, "sync id: %u", i);
344 		if (rc)
345 			goto free_sync_objects;
346 		sync_name = hl_state_dump_get_sync_name(hdev, i);
347 		if (sync_name) {
348 			rc = hl_snprintf_resize(buf, size, offset, " %s",
349 						sync_name);
350 			if (rc)
351 				goto free_sync_objects;
352 		}
353 		rc = hl_snprintf_resize(buf, size, offset, ", value: %u",
354 					sync_objects[i]);
355 		if (rc)
356 			goto free_sync_objects;
357 
358 		/* Append engine string */
359 		entry = hl_state_dump_get_sync_to_engine(map,
360 			(u32)sync_object_addr);
361 		if (entry) {
362 			rc = hl_snprintf_resize(buf, size, offset,
363 						", Engine: ");
364 			if (rc)
365 				goto free_sync_objects;
366 			rc = hl_print_resize_sync_engine(buf, size, offset,
367 						entry->engine_type,
368 						entry->engine_id);
369 			if (rc)
370 				goto free_sync_objects;
371 		}
372 
373 		rc = hl_snprintf_resize(buf, size, offset, "\n");
374 		if (rc)
375 			goto free_sync_objects;
376 	}
377 
378 free_sync_objects:
379 	hl_state_dump_free_sync_objects(sync_objects);
380 out:
381 	return rc;
382 }
383 
384 /**
385  * hl_state_dump_print_syncs - print active sync objects
386  * @hdev: pointer to the device
387  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
388  * @size: pointer to the size container
389  * @offset: pointer to the offset container
390  *
391  * Returns 0 on success or error code on failure
392  */
hl_state_dump_print_syncs(struct hl_device * hdev,char ** buf,size_t * size,size_t * offset)393 static int hl_state_dump_print_syncs(struct hl_device *hdev,
394 					char **buf, size_t *size,
395 					size_t *offset)
396 
397 {
398 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
399 	struct hl_sync_to_engine_map *map;
400 	u32 index;
401 	int rc = 0;
402 
403 	map = kzalloc(sizeof(*map), GFP_KERNEL);
404 	if (!map)
405 		return -ENOMEM;
406 
407 	rc = sds->funcs.gen_sync_to_engine_map(hdev, map);
408 	if (rc)
409 		goto free_map_mem;
410 
411 	rc = hl_snprintf_resize(buf, size, offset, "Non zero sync objects:\n");
412 	if (rc)
413 		goto out;
414 
415 	if (sds->sync_namager_names) {
416 		for (index = 0; sds->sync_namager_names[index]; ++index) {
417 			rc = hl_state_dump_print_syncs_single_block(
418 				hdev, index, buf, size, offset, map);
419 			if (rc)
420 				goto out;
421 		}
422 	} else {
423 		for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
424 			rc = hl_state_dump_print_syncs_single_block(
425 				hdev, index, buf, size, offset, map);
426 			if (rc)
427 				goto out;
428 		}
429 	}
430 
431 out:
432 	hl_state_dump_free_sync_to_engine_map(map);
433 free_map_mem:
434 	kfree(map);
435 
436 	return rc;
437 }
438 
439 /**
440  * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific
441  * block
442  * @hdev: pointer to the device
443  * @index: sync manager block index starting with E_N
444  *
445  * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL
446  * on error
447  */
448 static struct hl_mon_state_dump *
hl_state_dump_alloc_read_sm_block_monitors(struct hl_device * hdev,u32 index)449 hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index)
450 {
451 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
452 	struct hl_mon_state_dump *monitors;
453 	s64 base_addr; /* Base addr can be negative */
454 	int i;
455 
456 	monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] *
457 			   sizeof(struct hl_mon_state_dump));
458 	if (!monitors)
459 		return NULL;
460 
461 	base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;
462 
463 	for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
464 		monitors[i].id = i;
465 		monitors[i].wr_addr_low =
466 			RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] +
467 				i * sizeof(u32));
468 
469 		monitors[i].wr_addr_high =
470 			RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] +
471 				i * sizeof(u32));
472 
473 		monitors[i].wr_data =
474 			RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] +
475 				i * sizeof(u32));
476 
477 		monitors[i].arm_data =
478 			RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] +
479 				i * sizeof(u32));
480 
481 		monitors[i].status =
482 			RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] +
483 				i * sizeof(u32));
484 	}
485 
486 	return monitors;
487 }
488 
489 /**
490  * hl_state_dump_free_monitors - free the monitors structure
491  * @monitors: monitors array created with
492  *            hl_state_dump_alloc_read_sm_block_monitors
493  */
hl_state_dump_free_monitors(struct hl_mon_state_dump * monitors)494 static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors)
495 {
496 	vfree(monitors);
497 }
498 
499 /**
500  * hl_state_dump_print_monitors_single_block - print active monitors on a
501  * single block
502  * @hdev: pointer to the device
503  * @index: sync manager block index starting with E_N
504  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
505  * @size: pointer to the size container
506  * @offset: pointer to the offset container
507  *
508  * Returns 0 on success or error code on failure
509  */
hl_state_dump_print_monitors_single_block(struct hl_device * hdev,u32 index,char ** buf,size_t * size,size_t * offset)510 static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev,
511 						u32 index,
512 						char **buf, size_t *size,
513 						size_t *offset)
514 {
515 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
516 	struct hl_mon_state_dump *monitors = NULL;
517 	int rc = 0, i;
518 
519 	if (sds->sync_namager_names) {
520 		rc = hl_snprintf_resize(
521 			buf, size, offset, "%s\n",
522 			sds->sync_namager_names[index]);
523 		if (rc)
524 			goto out;
525 	}
526 
527 	monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index);
528 	if (!monitors) {
529 		rc = -ENOMEM;
530 		goto out;
531 	}
532 
533 	for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {
534 		if (!(sds->funcs.monitor_valid(&monitors[i])))
535 			continue;
536 
537 		/* Monitor is valid, dump it */
538 		rc = sds->funcs.print_single_monitor(buf, size, offset, hdev,
539 							&monitors[i]);
540 		if (rc)
541 			goto free_monitors;
542 
543 		hl_snprintf_resize(buf, size, offset, "\n");
544 	}
545 
546 free_monitors:
547 	hl_state_dump_free_monitors(monitors);
548 out:
549 	return rc;
550 }
551 
552 /**
553  * hl_state_dump_print_monitors - print active monitors
554  * @hdev: pointer to the device
555  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
556  * @size: pointer to the size container
557  * @offset: pointer to the offset container
558  *
559  * Returns 0 on success or error code on failure
560  */
hl_state_dump_print_monitors(struct hl_device * hdev,char ** buf,size_t * size,size_t * offset)561 static int hl_state_dump_print_monitors(struct hl_device *hdev,
562 					char **buf, size_t *size,
563 					size_t *offset)
564 {
565 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
566 	u32 index;
567 	int rc = 0;
568 
569 	rc = hl_snprintf_resize(buf, size, offset,
570 		"Valid (armed) monitor objects:\n");
571 	if (rc)
572 		goto out;
573 
574 	if (sds->sync_namager_names) {
575 		for (index = 0; sds->sync_namager_names[index]; ++index) {
576 			rc = hl_state_dump_print_monitors_single_block(
577 				hdev, index, buf, size, offset);
578 			if (rc)
579 				goto out;
580 		}
581 	} else {
582 		for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {
583 			rc = hl_state_dump_print_monitors_single_block(
584 				hdev, index, buf, size, offset);
585 			if (rc)
586 				goto out;
587 		}
588 	}
589 
590 out:
591 	return rc;
592 }
593 
594 /**
595  * hl_state_dump_print_engine_fences - print active fences for a specific
596  * engine
597  * @hdev: pointer to the device
598  * @engine_type: engine type to use
599  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
600  * @size: pointer to the size container
601  * @offset: pointer to the offset container
602  */
603 static int
hl_state_dump_print_engine_fences(struct hl_device * hdev,enum hl_sync_engine_type engine_type,char ** buf,size_t * size,size_t * offset)604 hl_state_dump_print_engine_fences(struct hl_device *hdev,
605 				  enum hl_sync_engine_type engine_type,
606 				  char **buf, size_t *size, size_t *offset)
607 {
608 	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;
609 	int rc = 0, i, n_fences;
610 	u64 base_addr, next_fence;
611 
612 	switch (engine_type) {
613 	case ENGINE_TPC:
614 		n_fences = sds->props[SP_NUM_OF_TPC_ENGINES];
615 		base_addr = sds->props[SP_TPC0_CMDQ];
616 		next_fence = sds->props[SP_NEXT_TPC];
617 		break;
618 	case ENGINE_MME:
619 		n_fences = sds->props[SP_NUM_OF_MME_ENGINES];
620 		base_addr = sds->props[SP_MME_CMDQ];
621 		next_fence = sds->props[SP_NEXT_MME];
622 		break;
623 	case ENGINE_DMA:
624 		n_fences = sds->props[SP_NUM_OF_DMA_ENGINES];
625 		base_addr = sds->props[SP_DMA_CMDQ];
626 		next_fence = sds->props[SP_DMA_QUEUES_OFFSET];
627 		break;
628 	default:
629 		return -EINVAL;
630 	}
631 	for (i = 0; i < n_fences; ++i) {
632 		rc = sds->funcs.print_fences_single_engine(
633 			hdev,
634 			base_addr + next_fence * i +
635 				sds->props[SP_FENCE0_CNT_OFFSET],
636 			base_addr + next_fence * i +
637 				sds->props[SP_CP_STS_OFFSET],
638 			engine_type, i, buf, size, offset);
639 		if (rc)
640 			goto out;
641 	}
642 out:
643 	return rc;
644 }
645 
646 /**
647  * hl_state_dump_print_fences - print active fences
648  * @hdev: pointer to the device
649  * @buf: destination buffer double pointer to be used with hl_snprintf_resize
650  * @size: pointer to the size container
651  * @offset: pointer to the offset container
652  */
hl_state_dump_print_fences(struct hl_device * hdev,char ** buf,size_t * size,size_t * offset)653 static int hl_state_dump_print_fences(struct hl_device *hdev, char **buf,
654 				      size_t *size, size_t *offset)
655 {
656 	int rc = 0;
657 
658 	rc = hl_snprintf_resize(buf, size, offset, "Valid (armed) fences:\n");
659 	if (rc)
660 		goto out;
661 
662 	rc = hl_state_dump_print_engine_fences(hdev, ENGINE_TPC, buf, size, offset);
663 	if (rc)
664 		goto out;
665 
666 	rc = hl_state_dump_print_engine_fences(hdev, ENGINE_MME, buf, size, offset);
667 	if (rc)
668 		goto out;
669 
670 	rc = hl_state_dump_print_engine_fences(hdev, ENGINE_DMA, buf, size, offset);
671 	if (rc)
672 		goto out;
673 
674 out:
675 	return rc;
676 }
677 
678 /**
679  * hl_state_dump() - dump system state
680  * @hdev: pointer to device structure
681  */
hl_state_dump(struct hl_device * hdev)682 int hl_state_dump(struct hl_device *hdev)
683 {
684 	char *buf = NULL;
685 	size_t offset = 0, size = 0;
686 	int rc;
687 
688 	rc = hl_snprintf_resize(&buf, &size, &offset,
689 				"Timestamp taken on: %llu\n\n",
690 				ktime_to_ns(ktime_get()));
691 	if (rc)
692 		goto err;
693 
694 	rc = hl_state_dump_print_syncs(hdev, &buf, &size, &offset);
695 	if (rc)
696 		goto err;
697 
698 	hl_snprintf_resize(&buf, &size, &offset, "\n");
699 
700 	rc = hl_state_dump_print_monitors(hdev, &buf, &size, &offset);
701 	if (rc)
702 		goto err;
703 
704 	hl_snprintf_resize(&buf, &size, &offset, "\n");
705 
706 	rc = hl_state_dump_print_fences(hdev, &buf, &size, &offset);
707 	if (rc)
708 		goto err;
709 
710 	hl_snprintf_resize(&buf, &size, &offset, "\n");
711 
712 	hl_debugfs_set_state_dump(hdev, buf, size);
713 
714 	return 0;
715 err:
716 	vfree(buf);
717 	return rc;
718 }
719