1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2016-2022 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7 
8 #define pr_fmt(fmt)			"habanalabs: " fmt
9 
10 #include <uapi/drm/habanalabs_accel.h>
11 #include "habanalabs.h"
12 
13 #include <linux/pci.h>
14 #include <linux/hwmon.h>
15 #include <linux/vmalloc.h>
16 
17 #include <trace/events/habanalabs.h>
18 
19 #define HL_RESET_DELAY_USEC			10000	/* 10ms */
20 
21 #define HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC	5
22 
23 enum dma_alloc_type {
24 	DMA_ALLOC_COHERENT,
25 	DMA_ALLOC_POOL,
26 };
27 
28 #define MEM_SCRUB_DEFAULT_VAL 0x1122334455667788
29 
30 /*
31  * hl_set_dram_bar- sets the bar to allow later access to address
32  *
33  * @hdev: pointer to habanalabs device structure.
34  * @addr: the address the caller wants to access.
35  * @region: the PCI region.
36  * @new_bar_region_base: the new BAR region base address.
37  *
38  * @return: the old BAR base address on success, U64_MAX for failure.
39  *	    The caller should set it back to the old address after use.
40  *
41  * In case the bar space does not cover the whole address space,
42  * the bar base address should be set to allow access to a given address.
43  * This function can be called also if the bar doesn't need to be set,
44  * in that case it just won't change the base.
45  */
46 static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region,
47 				u64 *new_bar_region_base)
48 {
49 	struct asic_fixed_properties *prop = &hdev->asic_prop;
50 	u64 bar_base_addr, old_base;
51 
52 	if (is_power_of_2(prop->dram_pci_bar_size))
53 		bar_base_addr = addr & ~(prop->dram_pci_bar_size - 0x1ull);
54 	else
55 		bar_base_addr = DIV_ROUND_DOWN_ULL(addr, prop->dram_pci_bar_size) *
56 				prop->dram_pci_bar_size;
57 
58 	old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr);
59 
60 	/* in case of success we need to update the new BAR base */
61 	if ((old_base != U64_MAX) && new_bar_region_base)
62 		*new_bar_region_base = bar_base_addr;
63 
64 	return old_base;
65 }
66 
67 int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val,
68 	enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar)
69 {
70 	struct pci_mem_region *region = &hdev->pci_mem_region[region_type];
71 	u64 old_base = 0, rc, bar_region_base = region->region_base;
72 	void __iomem *acc_addr;
73 
74 	if (set_dram_bar) {
75 		old_base = hl_set_dram_bar(hdev, addr, region, &bar_region_base);
76 		if (old_base == U64_MAX)
77 			return -EIO;
78 	}
79 
80 	acc_addr = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
81 			(addr - bar_region_base);
82 
83 	switch (acc_type) {
84 	case DEBUGFS_READ8:
85 		*val = readb(acc_addr);
86 		break;
87 	case DEBUGFS_WRITE8:
88 		writeb(*val, acc_addr);
89 		break;
90 	case DEBUGFS_READ32:
91 		*val = readl(acc_addr);
92 		break;
93 	case DEBUGFS_WRITE32:
94 		writel(*val, acc_addr);
95 		break;
96 	case DEBUGFS_READ64:
97 		*val = readq(acc_addr);
98 		break;
99 	case DEBUGFS_WRITE64:
100 		writeq(*val, acc_addr);
101 		break;
102 	}
103 
104 	if (set_dram_bar) {
105 		rc = hl_set_dram_bar(hdev, old_base, region, NULL);
106 		if (rc == U64_MAX)
107 			return -EIO;
108 	}
109 
110 	return 0;
111 }
112 
113 static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
114 					gfp_t flag, enum dma_alloc_type alloc_type,
115 					const char *caller)
116 {
117 	void *ptr = NULL;
118 
119 	switch (alloc_type) {
120 	case DMA_ALLOC_COHERENT:
121 		ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag);
122 		break;
123 	case DMA_ALLOC_POOL:
124 		ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle);
125 		break;
126 	}
127 
128 	if (trace_habanalabs_dma_alloc_enabled() && !ZERO_OR_NULL_PTR(ptr))
129 		trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size,
130 						caller);
131 
132 	return ptr;
133 }
134 
135 static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr,
136 					dma_addr_t dma_handle, enum dma_alloc_type alloc_type,
137 					const char *caller)
138 {
139 	/* this is needed to avoid warning on using freed pointer */
140 	u64 store_cpu_addr = (u64) (uintptr_t) cpu_addr;
141 
142 	switch (alloc_type) {
143 	case DMA_ALLOC_COHERENT:
144 		hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle);
145 		break;
146 	case DMA_ALLOC_POOL:
147 		hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle);
148 		break;
149 	}
150 
151 	trace_habanalabs_dma_free(hdev->dev, store_cpu_addr, dma_handle, size, caller);
152 }
153 
154 void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
155 					gfp_t flag, const char *caller)
156 {
157 	return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller);
158 }
159 
160 void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr,
161 					dma_addr_t dma_handle, const char *caller)
162 {
163 	hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller);
164 }
165 
166 void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags,
167 					dma_addr_t *dma_handle, const char *caller)
168 {
169 	return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller);
170 }
171 
172 void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr,
173 					const char *caller)
174 {
175 	hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller);
176 }
177 
178 void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle)
179 {
180 	return hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
181 }
182 
183 void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
184 {
185 	hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, vaddr);
186 }
187 
188 int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
189 {
190 	struct asic_fixed_properties *prop = &hdev->asic_prop;
191 	struct scatterlist *sg;
192 	int rc, i;
193 
194 	rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0);
195 	if (rc)
196 		return rc;
197 
198 	/* Shift to the device's base physical address of host memory if necessary */
199 	if (prop->device_dma_offset_for_host_access)
200 		for_each_sgtable_dma_sg(sgt, sg, i)
201 			sg->dma_address += prop->device_dma_offset_for_host_access;
202 
203 	return 0;
204 }
205 
206 void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
207 {
208 	struct asic_fixed_properties *prop = &hdev->asic_prop;
209 	struct scatterlist *sg;
210 	int i;
211 
212 	/* Cancel the device's base physical address of host memory if necessary */
213 	if (prop->device_dma_offset_for_host_access)
214 		for_each_sgtable_dma_sg(sgt, sg, i)
215 			sg->dma_address -= prop->device_dma_offset_for_host_access;
216 
217 	dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0);
218 }
219 
220 /*
221  * hl_access_cfg_region - access the config region
222  *
223  * @hdev: pointer to habanalabs device structure
224  * @addr: the address to access
225  * @val: the value to write from or read to
226  * @acc_type: the type of access (read/write 64/32)
227  */
228 int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val,
229 	enum debugfs_access_type acc_type)
230 {
231 	struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG];
232 	u32 val_h, val_l;
233 
234 	if (!IS_ALIGNED(addr, sizeof(u32))) {
235 		dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32));
236 		return -EINVAL;
237 	}
238 
239 	switch (acc_type) {
240 	case DEBUGFS_READ32:
241 		*val = RREG32(addr - cfg_region->region_base);
242 		break;
243 	case DEBUGFS_WRITE32:
244 		WREG32(addr - cfg_region->region_base, *val);
245 		break;
246 	case DEBUGFS_READ64:
247 		val_l = RREG32(addr - cfg_region->region_base);
248 		val_h = RREG32(addr + sizeof(u32) - cfg_region->region_base);
249 
250 		*val = (((u64) val_h) << 32) | val_l;
251 		break;
252 	case DEBUGFS_WRITE64:
253 		WREG32(addr - cfg_region->region_base, lower_32_bits(*val));
254 		WREG32(addr + sizeof(u32) - cfg_region->region_base, upper_32_bits(*val));
255 		break;
256 	default:
257 		dev_err(hdev->dev, "access type %d is not supported\n", acc_type);
258 		return -EOPNOTSUPP;
259 	}
260 
261 	return 0;
262 }
263 
264 /*
265  * hl_access_dev_mem - access device memory
266  *
267  * @hdev: pointer to habanalabs device structure
268  * @region_type: the type of the region the address belongs to
269  * @addr: the address to access
270  * @val: the value to write from or read to
271  * @acc_type: the type of access (r/w, 32/64)
272  */
273 int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type,
274 			u64 addr, u64 *val, enum debugfs_access_type acc_type)
275 {
276 	switch (region_type) {
277 	case PCI_REGION_CFG:
278 		return hl_access_cfg_region(hdev, addr, val, acc_type);
279 	case PCI_REGION_SRAM:
280 	case PCI_REGION_DRAM:
281 		return hl_access_sram_dram_region(hdev, addr, val, acc_type,
282 				region_type, (region_type == PCI_REGION_DRAM));
283 	default:
284 		return -EFAULT;
285 	}
286 
287 	return 0;
288 }
289 
290 void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...)
291 {
292 	va_list args;
293 	int str_size;
294 
295 	va_start(args, fmt);
296 	/* Calculate formatted string length. Assuming each string is null terminated, hence
297 	 * increment result by 1
298 	 */
299 	str_size = vsnprintf(NULL, 0, fmt, args) + 1;
300 	va_end(args);
301 
302 	if ((e->actual_size + str_size) < e->allocated_buf_size) {
303 		va_start(args, fmt);
304 		vsnprintf(e->buf + e->actual_size, str_size, fmt, args);
305 		va_end(args);
306 	}
307 
308 	/* Need to update the size even when not updating destination buffer to get the exact size
309 	 * of all input strings
310 	 */
311 	e->actual_size += str_size;
312 }
313 
314 enum hl_device_status hl_device_status(struct hl_device *hdev)
315 {
316 	enum hl_device_status status;
317 
318 	if (hdev->reset_info.in_reset) {
319 		if (hdev->reset_info.in_compute_reset)
320 			status = HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE;
321 		else
322 			status = HL_DEVICE_STATUS_IN_RESET;
323 	} else if (hdev->reset_info.needs_reset) {
324 		status = HL_DEVICE_STATUS_NEEDS_RESET;
325 	} else if (hdev->disabled) {
326 		status = HL_DEVICE_STATUS_MALFUNCTION;
327 	} else if (!hdev->init_done) {
328 		status = HL_DEVICE_STATUS_IN_DEVICE_CREATION;
329 	} else {
330 		status = HL_DEVICE_STATUS_OPERATIONAL;
331 	}
332 
333 	return status;
334 }
335 
336 bool hl_device_operational(struct hl_device *hdev,
337 		enum hl_device_status *status)
338 {
339 	enum hl_device_status current_status;
340 
341 	current_status = hl_device_status(hdev);
342 	if (status)
343 		*status = current_status;
344 
345 	switch (current_status) {
346 	case HL_DEVICE_STATUS_IN_RESET:
347 	case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE:
348 	case HL_DEVICE_STATUS_MALFUNCTION:
349 	case HL_DEVICE_STATUS_NEEDS_RESET:
350 		return false;
351 	case HL_DEVICE_STATUS_OPERATIONAL:
352 	case HL_DEVICE_STATUS_IN_DEVICE_CREATION:
353 	default:
354 		return true;
355 	}
356 }
357 
358 bool hl_ctrl_device_operational(struct hl_device *hdev,
359 		enum hl_device_status *status)
360 {
361 	enum hl_device_status current_status;
362 
363 	current_status = hl_device_status(hdev);
364 	if (status)
365 		*status = current_status;
366 
367 	switch (current_status) {
368 	case HL_DEVICE_STATUS_MALFUNCTION:
369 		return false;
370 	case HL_DEVICE_STATUS_IN_RESET:
371 	case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE:
372 	case HL_DEVICE_STATUS_NEEDS_RESET:
373 	case HL_DEVICE_STATUS_OPERATIONAL:
374 	case HL_DEVICE_STATUS_IN_DEVICE_CREATION:
375 	default:
376 		return true;
377 	}
378 }
379 
380 static void print_idle_status_mask(struct hl_device *hdev, const char *message,
381 					u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE])
382 {
383 	if (idle_mask[3])
384 		dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx_%016llx)\n",
385 			message, idle_mask[3], idle_mask[2], idle_mask[1], idle_mask[0]);
386 	else if (idle_mask[2])
387 		dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx)\n",
388 			message, idle_mask[2], idle_mask[1], idle_mask[0]);
389 	else if (idle_mask[1])
390 		dev_err(hdev->dev, "%s (mask %#llx_%016llx)\n",
391 			message, idle_mask[1], idle_mask[0]);
392 	else
393 		dev_err(hdev->dev, "%s (mask %#llx)\n", message, idle_mask[0]);
394 }
395 
396 static void hpriv_release(struct kref *ref)
397 {
398 	u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
399 	bool reset_device, device_is_idle = true;
400 	struct hl_fpriv *hpriv;
401 	struct hl_device *hdev;
402 
403 	hpriv = container_of(ref, struct hl_fpriv, refcount);
404 
405 	hdev = hpriv->hdev;
406 
407 	hdev->asic_funcs->send_device_activity(hdev, false);
408 
409 	put_pid(hpriv->taskpid);
410 
411 	hl_debugfs_remove_file(hpriv);
412 
413 	mutex_destroy(&hpriv->ctx_lock);
414 	mutex_destroy(&hpriv->restore_phase_mutex);
415 
416 	/* There should be no memory buffers at this point and handles IDR can be destroyed */
417 	hl_mem_mgr_idr_destroy(&hpriv->mem_mgr);
418 
419 	/* Device should be reset if reset-upon-device-release is enabled, or if there is a pending
420 	 * reset that waits for device release.
421 	 */
422 	reset_device = hdev->reset_upon_device_release || hdev->reset_info.watchdog_active;
423 
424 	/* Check the device idle status and reset if not idle.
425 	 * Skip it if already in reset, or if device is going to be reset in any case.
426 	 */
427 	if (!hdev->reset_info.in_reset && !reset_device && hdev->pdev && !hdev->pldm)
428 		device_is_idle = hdev->asic_funcs->is_device_idle(hdev, idle_mask,
429 							HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL);
430 	if (!device_is_idle) {
431 		print_idle_status_mask(hdev, "device is not idle after user context is closed",
432 					idle_mask);
433 		reset_device = true;
434 	}
435 
436 	/* We need to remove the user from the list to make sure the reset process won't
437 	 * try to kill the user process. Because, if we got here, it means there are no
438 	 * more driver/device resources that the user process is occupying so there is
439 	 * no need to kill it
440 	 *
441 	 * However, we can't set the compute_ctx to NULL at this stage. This is to prevent
442 	 * a race between the release and opening the device again. We don't want to let
443 	 * a user open the device while there a reset is about to happen.
444 	 */
445 	mutex_lock(&hdev->fpriv_list_lock);
446 	list_del(&hpriv->dev_node);
447 	mutex_unlock(&hdev->fpriv_list_lock);
448 
449 	if (reset_device) {
450 		hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE);
451 	} else {
452 		/* Scrubbing is handled within hl_device_reset(), so here need to do it directly */
453 		int rc = hdev->asic_funcs->scrub_device_mem(hdev);
454 
455 		if (rc)
456 			dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc);
457 	}
458 
459 	/* Now we can mark the compute_ctx as not active. Even if a reset is running in a different
460 	 * thread, we don't care because the in_reset is marked so if a user will try to open
461 	 * the device it will fail on that, even if compute_ctx is false.
462 	 */
463 	mutex_lock(&hdev->fpriv_list_lock);
464 	hdev->is_compute_ctx_active = false;
465 	mutex_unlock(&hdev->fpriv_list_lock);
466 
467 	hdev->compute_ctx_in_release = 0;
468 
469 	/* release the eventfd */
470 	if (hpriv->notifier_event.eventfd)
471 		eventfd_ctx_put(hpriv->notifier_event.eventfd);
472 
473 	mutex_destroy(&hpriv->notifier_event.lock);
474 
475 	kfree(hpriv);
476 }
477 
478 void hl_hpriv_get(struct hl_fpriv *hpriv)
479 {
480 	kref_get(&hpriv->refcount);
481 }
482 
483 int hl_hpriv_put(struct hl_fpriv *hpriv)
484 {
485 	return kref_put(&hpriv->refcount, hpriv_release);
486 }
487 
488 static void print_device_in_use_info(struct hl_device *hdev, const char *message)
489 {
490 	u32 active_cs_num, dmabuf_export_cnt;
491 	bool unknown_reason = true;
492 	char buf[128];
493 	size_t size;
494 	int offset;
495 
496 	size = sizeof(buf);
497 	offset = 0;
498 
499 	active_cs_num = hl_get_active_cs_num(hdev);
500 	if (active_cs_num) {
501 		unknown_reason = false;
502 		offset += scnprintf(buf + offset, size - offset, " [%u active CS]", active_cs_num);
503 	}
504 
505 	dmabuf_export_cnt = atomic_read(&hdev->dmabuf_export_cnt);
506 	if (dmabuf_export_cnt) {
507 		unknown_reason = false;
508 		offset += scnprintf(buf + offset, size - offset, " [%u exported dma-buf]",
509 					dmabuf_export_cnt);
510 	}
511 
512 	if (unknown_reason)
513 		scnprintf(buf + offset, size - offset, " [unknown reason]");
514 
515 	dev_notice(hdev->dev, "%s%s\n", message, buf);
516 }
517 
518 /*
519  * hl_device_release - release function for habanalabs device
520  *
521  * @inode: pointer to inode structure
522  * @filp: pointer to file structure
523  *
524  * Called when process closes an habanalabs device
525  */
526 static int hl_device_release(struct inode *inode, struct file *filp)
527 {
528 	struct hl_fpriv *hpriv = filp->private_data;
529 	struct hl_device *hdev = hpriv->hdev;
530 
531 	filp->private_data = NULL;
532 
533 	if (!hdev) {
534 		pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n");
535 		put_pid(hpriv->taskpid);
536 		return 0;
537 	}
538 
539 	hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
540 
541 	/* Memory buffers might be still in use at this point and thus the handles IDR destruction
542 	 * is postponed to hpriv_release().
543 	 */
544 	hl_mem_mgr_fini(&hpriv->mem_mgr);
545 
546 	hdev->compute_ctx_in_release = 1;
547 
548 	if (!hl_hpriv_put(hpriv)) {
549 		print_device_in_use_info(hdev, "User process closed FD but device still in use");
550 		hl_device_reset(hdev, HL_DRV_RESET_HARD);
551 	}
552 
553 	hdev->last_open_session_duration_jif = jiffies - hdev->last_successful_open_jif;
554 
555 	return 0;
556 }
557 
558 static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
559 {
560 	struct hl_fpriv *hpriv = filp->private_data;
561 	struct hl_device *hdev = hpriv->hdev;
562 
563 	filp->private_data = NULL;
564 
565 	if (!hdev) {
566 		pr_err("Closing FD after device was removed\n");
567 		goto out;
568 	}
569 
570 	mutex_lock(&hdev->fpriv_ctrl_list_lock);
571 	list_del(&hpriv->dev_node);
572 	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
573 out:
574 	/* release the eventfd */
575 	if (hpriv->notifier_event.eventfd)
576 		eventfd_ctx_put(hpriv->notifier_event.eventfd);
577 
578 	mutex_destroy(&hpriv->notifier_event.lock);
579 	put_pid(hpriv->taskpid);
580 
581 	kfree(hpriv);
582 
583 	return 0;
584 }
585 
586 /*
587  * hl_mmap - mmap function for habanalabs device
588  *
589  * @*filp: pointer to file structure
590  * @*vma: pointer to vm_area_struct of the process
591  *
592  * Called when process does an mmap on habanalabs device. Call the relevant mmap
593  * function at the end of the common code.
594  */
595 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
596 {
597 	struct hl_fpriv *hpriv = filp->private_data;
598 	struct hl_device *hdev = hpriv->hdev;
599 	unsigned long vm_pgoff;
600 
601 	if (!hdev) {
602 		pr_err_ratelimited("Trying to mmap after device was removed! Please close FD\n");
603 		return -ENODEV;
604 	}
605 
606 	vm_pgoff = vma->vm_pgoff;
607 
608 	switch (vm_pgoff & HL_MMAP_TYPE_MASK) {
609 	case HL_MMAP_TYPE_BLOCK:
610 		vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff);
611 		return hl_hw_block_mmap(hpriv, vma);
612 
613 	case HL_MMAP_TYPE_CB:
614 	case HL_MMAP_TYPE_TS_BUFF:
615 		return hl_mem_mgr_mmap(&hpriv->mem_mgr, vma, NULL);
616 	}
617 	return -EINVAL;
618 }
619 
620 static const struct file_operations hl_ops = {
621 	.owner = THIS_MODULE,
622 	.open = hl_device_open,
623 	.release = hl_device_release,
624 	.mmap = hl_mmap,
625 	.unlocked_ioctl = hl_ioctl,
626 	.compat_ioctl = hl_ioctl
627 };
628 
629 static const struct file_operations hl_ctrl_ops = {
630 	.owner = THIS_MODULE,
631 	.open = hl_device_open_ctrl,
632 	.release = hl_device_release_ctrl,
633 	.unlocked_ioctl = hl_ioctl_control,
634 	.compat_ioctl = hl_ioctl_control
635 };
636 
637 static void device_release_func(struct device *dev)
638 {
639 	kfree(dev);
640 }
641 
642 /*
643  * device_init_cdev - Initialize cdev and device for habanalabs device
644  *
645  * @hdev: pointer to habanalabs device structure
646  * @class: pointer to the class object of the device
647  * @minor: minor number of the specific device
648  * @fpos: file operations to install for this device
649  * @name: name of the device as it will appear in the filesystem
650  * @cdev: pointer to the char device object that will be initialized
651  * @dev: pointer to the device object that will be initialized
652  *
653  * Initialize a cdev and a Linux device for habanalabs's device.
654  */
655 static int device_init_cdev(struct hl_device *hdev, struct class *class,
656 				int minor, const struct file_operations *fops,
657 				char *name, struct cdev *cdev,
658 				struct device **dev)
659 {
660 	cdev_init(cdev, fops);
661 	cdev->owner = THIS_MODULE;
662 
663 	*dev = kzalloc(sizeof(**dev), GFP_KERNEL);
664 	if (!*dev)
665 		return -ENOMEM;
666 
667 	device_initialize(*dev);
668 	(*dev)->devt = MKDEV(hdev->major, minor);
669 	(*dev)->class = class;
670 	(*dev)->release = device_release_func;
671 	dev_set_drvdata(*dev, hdev);
672 	dev_set_name(*dev, "%s", name);
673 
674 	return 0;
675 }
676 
677 static int cdev_sysfs_debugfs_add(struct hl_device *hdev)
678 {
679 	int rc;
680 
681 	rc = cdev_device_add(&hdev->cdev, hdev->dev);
682 	if (rc) {
683 		dev_err(hdev->dev,
684 			"failed to add a char device to the system\n");
685 		return rc;
686 	}
687 
688 	rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
689 	if (rc) {
690 		dev_err(hdev->dev,
691 			"failed to add a control char device to the system\n");
692 		goto delete_cdev_device;
693 	}
694 
695 	/* hl_sysfs_init() must be done after adding the device to the system */
696 	rc = hl_sysfs_init(hdev);
697 	if (rc) {
698 		dev_err(hdev->dev, "failed to initialize sysfs\n");
699 		goto delete_ctrl_cdev_device;
700 	}
701 
702 	hl_debugfs_add_device(hdev);
703 
704 	hdev->cdev_sysfs_debugfs_created = true;
705 
706 	return 0;
707 
708 delete_ctrl_cdev_device:
709 	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
710 delete_cdev_device:
711 	cdev_device_del(&hdev->cdev, hdev->dev);
712 	return rc;
713 }
714 
715 static void cdev_sysfs_debugfs_remove(struct hl_device *hdev)
716 {
717 	if (!hdev->cdev_sysfs_debugfs_created)
718 		goto put_devices;
719 
720 	hl_debugfs_remove_device(hdev);
721 	hl_sysfs_fini(hdev);
722 	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
723 	cdev_device_del(&hdev->cdev, hdev->dev);
724 
725 put_devices:
726 	put_device(hdev->dev);
727 	put_device(hdev->dev_ctrl);
728 }
729 
730 static void device_hard_reset_pending(struct work_struct *work)
731 {
732 	struct hl_device_reset_work *device_reset_work =
733 		container_of(work, struct hl_device_reset_work, reset_work.work);
734 	struct hl_device *hdev = device_reset_work->hdev;
735 	u32 flags;
736 	int rc;
737 
738 	flags = device_reset_work->flags | HL_DRV_RESET_FROM_RESET_THR;
739 
740 	rc = hl_device_reset(hdev, flags);
741 
742 	if ((rc == -EBUSY) && !hdev->device_fini_pending) {
743 		struct hl_ctx *ctx = hl_get_compute_ctx(hdev);
744 
745 		if (ctx) {
746 			/* The read refcount value should subtracted by one, because the read is
747 			 * protected with hl_get_compute_ctx().
748 			 */
749 			dev_info(hdev->dev,
750 				"Could not reset device (compute_ctx refcount %u). will try again in %u seconds",
751 				kref_read(&ctx->refcount) - 1, HL_PENDING_RESET_PER_SEC);
752 			hl_ctx_put(ctx);
753 		} else {
754 			dev_info(hdev->dev, "Could not reset device. will try again in %u seconds",
755 				HL_PENDING_RESET_PER_SEC);
756 		}
757 
758 		queue_delayed_work(hdev->reset_wq, &device_reset_work->reset_work,
759 					msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000));
760 	}
761 }
762 
763 static void device_release_watchdog_func(struct work_struct *work)
764 {
765 	struct hl_device_reset_work *watchdog_work =
766 			container_of(work, struct hl_device_reset_work, reset_work.work);
767 	struct hl_device *hdev = watchdog_work->hdev;
768 	u32 flags;
769 
770 	dev_dbg(hdev->dev, "Device wasn't released in time. Initiate hard-reset.\n");
771 
772 	flags = watchdog_work->flags | HL_DRV_RESET_HARD | HL_DRV_RESET_FROM_WD_THR;
773 
774 	hl_device_reset(hdev, flags);
775 }
776 
777 /*
778  * device_early_init - do some early initialization for the habanalabs device
779  *
780  * @hdev: pointer to habanalabs device structure
781  *
782  * Install the relevant function pointers and call the early_init function,
783  * if such a function exists
784  */
785 static int device_early_init(struct hl_device *hdev)
786 {
787 	int i, rc;
788 	char workq_name[32];
789 
790 	switch (hdev->asic_type) {
791 	case ASIC_GOYA:
792 		goya_set_asic_funcs(hdev);
793 		strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
794 		break;
795 	case ASIC_GAUDI:
796 		gaudi_set_asic_funcs(hdev);
797 		strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name));
798 		break;
799 	case ASIC_GAUDI_SEC:
800 		gaudi_set_asic_funcs(hdev);
801 		strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name));
802 		break;
803 	case ASIC_GAUDI2:
804 		gaudi2_set_asic_funcs(hdev);
805 		strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name));
806 		break;
807 	case ASIC_GAUDI2B:
808 		gaudi2_set_asic_funcs(hdev);
809 		strscpy(hdev->asic_name, "GAUDI2B", sizeof(hdev->asic_name));
810 		break;
811 	case ASIC_GAUDI2C:
812 		gaudi2_set_asic_funcs(hdev);
813 		strscpy(hdev->asic_name, "GAUDI2C", sizeof(hdev->asic_name));
814 		break;
815 	default:
816 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
817 			hdev->asic_type);
818 		return -EINVAL;
819 	}
820 
821 	rc = hdev->asic_funcs->early_init(hdev);
822 	if (rc)
823 		return rc;
824 
825 	rc = hl_asid_init(hdev);
826 	if (rc)
827 		goto early_fini;
828 
829 	if (hdev->asic_prop.completion_queues_count) {
830 		hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
831 				sizeof(struct workqueue_struct *),
832 				GFP_KERNEL);
833 		if (!hdev->cq_wq) {
834 			rc = -ENOMEM;
835 			goto asid_fini;
836 		}
837 	}
838 
839 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
840 		snprintf(workq_name, 32, "hl%u-free-jobs-%u", hdev->cdev_idx, (u32) i);
841 		hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
842 		if (hdev->cq_wq[i] == NULL) {
843 			dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
844 			rc = -ENOMEM;
845 			goto free_cq_wq;
846 		}
847 	}
848 
849 	snprintf(workq_name, 32, "hl%u-events", hdev->cdev_idx);
850 	hdev->eq_wq = create_singlethread_workqueue(workq_name);
851 	if (hdev->eq_wq == NULL) {
852 		dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
853 		rc = -ENOMEM;
854 		goto free_cq_wq;
855 	}
856 
857 	snprintf(workq_name, 32, "hl%u-cs-completions", hdev->cdev_idx);
858 	hdev->cs_cmplt_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
859 	if (!hdev->cs_cmplt_wq) {
860 		dev_err(hdev->dev,
861 			"Failed to allocate CS completions workqueue\n");
862 		rc = -ENOMEM;
863 		goto free_eq_wq;
864 	}
865 
866 	snprintf(workq_name, 32, "hl%u-ts-free-obj", hdev->cdev_idx);
867 	hdev->ts_free_obj_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
868 	if (!hdev->ts_free_obj_wq) {
869 		dev_err(hdev->dev,
870 			"Failed to allocate Timestamp registration free workqueue\n");
871 		rc = -ENOMEM;
872 		goto free_cs_cmplt_wq;
873 	}
874 
875 	snprintf(workq_name, 32, "hl%u-prefetch", hdev->cdev_idx);
876 	hdev->prefetch_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
877 	if (!hdev->prefetch_wq) {
878 		dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n");
879 		rc = -ENOMEM;
880 		goto free_ts_free_wq;
881 	}
882 
883 	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), GFP_KERNEL);
884 	if (!hdev->hl_chip_info) {
885 		rc = -ENOMEM;
886 		goto free_prefetch_wq;
887 	}
888 
889 	rc = hl_mmu_if_set_funcs(hdev);
890 	if (rc)
891 		goto free_chip_info;
892 
893 	hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr);
894 
895 	snprintf(workq_name, 32, "hl%u_device_reset", hdev->cdev_idx);
896 	hdev->reset_wq = create_singlethread_workqueue(workq_name);
897 	if (!hdev->reset_wq) {
898 		rc = -ENOMEM;
899 		dev_err(hdev->dev, "Failed to create device reset WQ\n");
900 		goto free_cb_mgr;
901 	}
902 
903 	INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, device_hard_reset_pending);
904 	hdev->device_reset_work.hdev = hdev;
905 	hdev->device_fini_pending = 0;
906 
907 	INIT_DELAYED_WORK(&hdev->device_release_watchdog_work.reset_work,
908 				device_release_watchdog_func);
909 	hdev->device_release_watchdog_work.hdev = hdev;
910 
911 	mutex_init(&hdev->send_cpu_message_lock);
912 	mutex_init(&hdev->debug_lock);
913 	INIT_LIST_HEAD(&hdev->cs_mirror_list);
914 	spin_lock_init(&hdev->cs_mirror_lock);
915 	spin_lock_init(&hdev->reset_info.lock);
916 	INIT_LIST_HEAD(&hdev->fpriv_list);
917 	INIT_LIST_HEAD(&hdev->fpriv_ctrl_list);
918 	mutex_init(&hdev->fpriv_list_lock);
919 	mutex_init(&hdev->fpriv_ctrl_list_lock);
920 	mutex_init(&hdev->clk_throttling.lock);
921 
922 	return 0;
923 
924 free_cb_mgr:
925 	hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
926 	hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
927 free_chip_info:
928 	kfree(hdev->hl_chip_info);
929 free_prefetch_wq:
930 	destroy_workqueue(hdev->prefetch_wq);
931 free_ts_free_wq:
932 	destroy_workqueue(hdev->ts_free_obj_wq);
933 free_cs_cmplt_wq:
934 	destroy_workqueue(hdev->cs_cmplt_wq);
935 free_eq_wq:
936 	destroy_workqueue(hdev->eq_wq);
937 free_cq_wq:
938 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
939 		if (hdev->cq_wq[i])
940 			destroy_workqueue(hdev->cq_wq[i]);
941 	kfree(hdev->cq_wq);
942 asid_fini:
943 	hl_asid_fini(hdev);
944 early_fini:
945 	if (hdev->asic_funcs->early_fini)
946 		hdev->asic_funcs->early_fini(hdev);
947 
948 	return rc;
949 }
950 
951 /*
952  * device_early_fini - finalize all that was done in device_early_init
953  *
954  * @hdev: pointer to habanalabs device structure
955  *
956  */
957 static void device_early_fini(struct hl_device *hdev)
958 {
959 	int i;
960 
961 	mutex_destroy(&hdev->debug_lock);
962 	mutex_destroy(&hdev->send_cpu_message_lock);
963 
964 	mutex_destroy(&hdev->fpriv_list_lock);
965 	mutex_destroy(&hdev->fpriv_ctrl_list_lock);
966 
967 	mutex_destroy(&hdev->clk_throttling.lock);
968 
969 	hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
970 	hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
971 
972 	kfree(hdev->hl_chip_info);
973 
974 	destroy_workqueue(hdev->prefetch_wq);
975 	destroy_workqueue(hdev->ts_free_obj_wq);
976 	destroy_workqueue(hdev->cs_cmplt_wq);
977 	destroy_workqueue(hdev->eq_wq);
978 	destroy_workqueue(hdev->reset_wq);
979 
980 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
981 		destroy_workqueue(hdev->cq_wq[i]);
982 	kfree(hdev->cq_wq);
983 
984 	hl_asid_fini(hdev);
985 
986 	if (hdev->asic_funcs->early_fini)
987 		hdev->asic_funcs->early_fini(hdev);
988 }
989 
990 static bool is_pci_link_healthy(struct hl_device *hdev)
991 {
992 	u16 vendor_id;
993 
994 	if (!hdev->pdev)
995 		return false;
996 
997 	pci_read_config_word(hdev->pdev, PCI_VENDOR_ID, &vendor_id);
998 
999 	return (vendor_id == PCI_VENDOR_ID_HABANALABS);
1000 }
1001 
1002 static void hl_device_heartbeat(struct work_struct *work)
1003 {
1004 	struct hl_device *hdev = container_of(work, struct hl_device,
1005 						work_heartbeat.work);
1006 	struct hl_info_fw_err_info info = {0};
1007 	u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
1008 
1009 	if (!hl_device_operational(hdev, NULL))
1010 		goto reschedule;
1011 
1012 	if (!hdev->asic_funcs->send_heartbeat(hdev))
1013 		goto reschedule;
1014 
1015 	if (hl_device_operational(hdev, NULL))
1016 		dev_err(hdev->dev, "Device heartbeat failed! PCI link is %s\n",
1017 			is_pci_link_healthy(hdev) ? "healthy" : "broken");
1018 
1019 	info.err_type = HL_INFO_FW_HEARTBEAT_ERR;
1020 	info.event_mask = &event_mask;
1021 	hl_handle_fw_err(hdev, &info);
1022 	hl_device_cond_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT, event_mask);
1023 
1024 	return;
1025 
1026 reschedule:
1027 	/*
1028 	 * prev_reset_trigger tracks consecutive fatal h/w errors until first
1029 	 * heartbeat immediately post reset.
1030 	 * If control reached here, then at least one heartbeat work has been
1031 	 * scheduled since last reset/init cycle.
1032 	 * So if the device is not already in reset cycle, reset the flag
1033 	 * prev_reset_trigger as no reset occurred with HL_DRV_RESET_FW_FATAL_ERR
1034 	 * status for at least one heartbeat. From this point driver restarts
1035 	 * tracking future consecutive fatal errors.
1036 	 */
1037 	if (!hdev->reset_info.in_reset)
1038 		hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
1039 
1040 	schedule_delayed_work(&hdev->work_heartbeat,
1041 			usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
1042 }
1043 
1044 /*
1045  * device_late_init - do late stuff initialization for the habanalabs device
1046  *
1047  * @hdev: pointer to habanalabs device structure
1048  *
1049  * Do stuff that either needs the device H/W queues to be active or needs
1050  * to happen after all the rest of the initialization is finished
1051  */
1052 static int device_late_init(struct hl_device *hdev)
1053 {
1054 	int rc;
1055 
1056 	if (hdev->asic_funcs->late_init) {
1057 		rc = hdev->asic_funcs->late_init(hdev);
1058 		if (rc) {
1059 			dev_err(hdev->dev,
1060 				"failed late initialization for the H/W\n");
1061 			return rc;
1062 		}
1063 	}
1064 
1065 	hdev->high_pll = hdev->asic_prop.high_pll;
1066 
1067 	if (hdev->heartbeat) {
1068 		INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
1069 		schedule_delayed_work(&hdev->work_heartbeat,
1070 				usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
1071 	}
1072 
1073 	hdev->late_init_done = true;
1074 
1075 	return 0;
1076 }
1077 
1078 /*
1079  * device_late_fini - finalize all that was done in device_late_init
1080  *
1081  * @hdev: pointer to habanalabs device structure
1082  *
1083  */
1084 static void device_late_fini(struct hl_device *hdev)
1085 {
1086 	if (!hdev->late_init_done)
1087 		return;
1088 
1089 	if (hdev->heartbeat)
1090 		cancel_delayed_work_sync(&hdev->work_heartbeat);
1091 
1092 	if (hdev->asic_funcs->late_fini)
1093 		hdev->asic_funcs->late_fini(hdev);
1094 
1095 	hdev->late_init_done = false;
1096 }
1097 
1098 int hl_device_utilization(struct hl_device *hdev, u32 *utilization)
1099 {
1100 	u64 max_power, curr_power, dc_power, dividend, divisor;
1101 	int rc;
1102 
1103 	max_power = hdev->max_power;
1104 	dc_power = hdev->asic_prop.dc_power_default;
1105 	divisor = max_power - dc_power;
1106 	if (!divisor) {
1107 		dev_warn(hdev->dev, "device utilization is not supported\n");
1108 		return -EOPNOTSUPP;
1109 	}
1110 	rc = hl_fw_cpucp_power_get(hdev, &curr_power);
1111 
1112 	if (rc)
1113 		return rc;
1114 
1115 	curr_power = clamp(curr_power, dc_power, max_power);
1116 
1117 	dividend = (curr_power - dc_power) * 100;
1118 	*utilization = (u32) div_u64(dividend, divisor);
1119 
1120 	return 0;
1121 }
1122 
1123 int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable)
1124 {
1125 	int rc = 0;
1126 
1127 	mutex_lock(&hdev->debug_lock);
1128 
1129 	if (!enable) {
1130 		if (!hdev->in_debug) {
1131 			dev_err(hdev->dev,
1132 				"Failed to disable debug mode because device was not in debug mode\n");
1133 			rc = -EFAULT;
1134 			goto out;
1135 		}
1136 
1137 		if (!hdev->reset_info.hard_reset_pending)
1138 			hdev->asic_funcs->halt_coresight(hdev, ctx);
1139 
1140 		hdev->in_debug = 0;
1141 
1142 		goto out;
1143 	}
1144 
1145 	if (hdev->in_debug) {
1146 		dev_err(hdev->dev,
1147 			"Failed to enable debug mode because device is already in debug mode\n");
1148 		rc = -EFAULT;
1149 		goto out;
1150 	}
1151 
1152 	hdev->in_debug = 1;
1153 
1154 out:
1155 	mutex_unlock(&hdev->debug_lock);
1156 
1157 	return rc;
1158 }
1159 
1160 static void take_release_locks(struct hl_device *hdev)
1161 {
1162 	/* Flush anyone that is inside the critical section of enqueue
1163 	 * jobs to the H/W
1164 	 */
1165 	hdev->asic_funcs->hw_queues_lock(hdev);
1166 	hdev->asic_funcs->hw_queues_unlock(hdev);
1167 
1168 	/* Flush processes that are sending message to CPU */
1169 	mutex_lock(&hdev->send_cpu_message_lock);
1170 	mutex_unlock(&hdev->send_cpu_message_lock);
1171 
1172 	/* Flush anyone that is inside device open */
1173 	mutex_lock(&hdev->fpriv_list_lock);
1174 	mutex_unlock(&hdev->fpriv_list_lock);
1175 	mutex_lock(&hdev->fpriv_ctrl_list_lock);
1176 	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
1177 }
1178 
1179 static void hl_abort_waiting_for_completions(struct hl_device *hdev)
1180 {
1181 	hl_abort_waiting_for_cs_completions(hdev);
1182 
1183 	/* Release all pending user interrupts, each pending user interrupt
1184 	 * holds a reference to a user context.
1185 	 */
1186 	hl_release_pending_user_interrupts(hdev);
1187 }
1188 
1189 static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset,
1190 				bool skip_wq_flush)
1191 {
1192 	if (hard_reset)
1193 		device_late_fini(hdev);
1194 
1195 	/*
1196 	 * Halt the engines and disable interrupts so we won't get any more
1197 	 * completions from H/W and we won't have any accesses from the
1198 	 * H/W to the host machine
1199 	 */
1200 	hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset);
1201 
1202 	/* Go over all the queues, release all CS and their jobs */
1203 	hl_cs_rollback_all(hdev, skip_wq_flush);
1204 
1205 	/* flush the MMU prefetch workqueue */
1206 	flush_workqueue(hdev->prefetch_wq);
1207 
1208 	hl_abort_waiting_for_completions(hdev);
1209 }
1210 
1211 /*
1212  * hl_device_suspend - initiate device suspend
1213  *
1214  * @hdev: pointer to habanalabs device structure
1215  *
1216  * Puts the hw in the suspend state (all asics).
1217  * Returns 0 for success or an error on failure.
1218  * Called at driver suspend.
1219  */
1220 int hl_device_suspend(struct hl_device *hdev)
1221 {
1222 	int rc;
1223 
1224 	pci_save_state(hdev->pdev);
1225 
1226 	/* Block future CS/VM/JOB completion operations */
1227 	spin_lock(&hdev->reset_info.lock);
1228 	if (hdev->reset_info.in_reset) {
1229 		spin_unlock(&hdev->reset_info.lock);
1230 		dev_err(hdev->dev, "Can't suspend while in reset\n");
1231 		return -EIO;
1232 	}
1233 	hdev->reset_info.in_reset = 1;
1234 	spin_unlock(&hdev->reset_info.lock);
1235 
1236 	/* This blocks all other stuff that is not blocked by in_reset */
1237 	hdev->disabled = true;
1238 
1239 	take_release_locks(hdev);
1240 
1241 	rc = hdev->asic_funcs->suspend(hdev);
1242 	if (rc)
1243 		dev_err(hdev->dev,
1244 			"Failed to disable PCI access of device CPU\n");
1245 
1246 	/* Shut down the device */
1247 	pci_disable_device(hdev->pdev);
1248 	pci_set_power_state(hdev->pdev, PCI_D3hot);
1249 
1250 	return 0;
1251 }
1252 
1253 /*
1254  * hl_device_resume - initiate device resume
1255  *
1256  * @hdev: pointer to habanalabs device structure
1257  *
1258  * Bring the hw back to operating state (all asics).
1259  * Returns 0 for success or an error on failure.
1260  * Called at driver resume.
1261  */
1262 int hl_device_resume(struct hl_device *hdev)
1263 {
1264 	int rc;
1265 
1266 	pci_set_power_state(hdev->pdev, PCI_D0);
1267 	pci_restore_state(hdev->pdev);
1268 	rc = pci_enable_device_mem(hdev->pdev);
1269 	if (rc) {
1270 		dev_err(hdev->dev,
1271 			"Failed to enable PCI device in resume\n");
1272 		return rc;
1273 	}
1274 
1275 	pci_set_master(hdev->pdev);
1276 
1277 	rc = hdev->asic_funcs->resume(hdev);
1278 	if (rc) {
1279 		dev_err(hdev->dev, "Failed to resume device after suspend\n");
1280 		goto disable_device;
1281 	}
1282 
1283 
1284 	/* 'in_reset' was set to true during suspend, now we must clear it in order
1285 	 * for hard reset to be performed
1286 	 */
1287 	spin_lock(&hdev->reset_info.lock);
1288 	hdev->reset_info.in_reset = 0;
1289 	spin_unlock(&hdev->reset_info.lock);
1290 
1291 	rc = hl_device_reset(hdev, HL_DRV_RESET_HARD);
1292 	if (rc) {
1293 		dev_err(hdev->dev, "Failed to reset device during resume\n");
1294 		goto disable_device;
1295 	}
1296 
1297 	return 0;
1298 
1299 disable_device:
1300 	pci_disable_device(hdev->pdev);
1301 
1302 	return rc;
1303 }
1304 
1305 static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev)
1306 {
1307 	struct task_struct *task = NULL;
1308 	struct list_head *fd_list;
1309 	struct hl_fpriv	*hpriv;
1310 	struct mutex *fd_lock;
1311 	u32 pending_cnt;
1312 
1313 	fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
1314 	fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
1315 
1316 	/* Giving time for user to close FD, and for processes that are inside
1317 	 * hl_device_open to finish
1318 	 */
1319 	if (!list_empty(fd_list))
1320 		ssleep(1);
1321 
1322 	if (timeout) {
1323 		pending_cnt = timeout;
1324 	} else {
1325 		if (hdev->process_kill_trial_cnt) {
1326 			/* Processes have been already killed */
1327 			pending_cnt = 1;
1328 			goto wait_for_processes;
1329 		} else {
1330 			/* Wait a small period after process kill */
1331 			pending_cnt = HL_PENDING_RESET_PER_SEC;
1332 		}
1333 	}
1334 
1335 	mutex_lock(fd_lock);
1336 
1337 	/* This section must be protected because we are dereferencing
1338 	 * pointers that are freed if the process exits
1339 	 */
1340 	list_for_each_entry(hpriv, fd_list, dev_node) {
1341 		task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
1342 		if (task) {
1343 			dev_info(hdev->dev, "Killing user process pid=%d\n",
1344 				task_pid_nr(task));
1345 			send_sig(SIGKILL, task, 1);
1346 			usleep_range(1000, 10000);
1347 
1348 			put_task_struct(task);
1349 		} else {
1350 			/*
1351 			 * If we got here, it means that process was killed from outside the driver
1352 			 * right after it started looping on fd_list and before get_pid_task, thus
1353 			 * we don't need to kill it.
1354 			 */
1355 			dev_dbg(hdev->dev,
1356 				"Can't get task struct for user process, assuming process was killed from outside the driver\n");
1357 		}
1358 	}
1359 
1360 	mutex_unlock(fd_lock);
1361 
1362 	/*
1363 	 * We killed the open users, but that doesn't mean they are closed.
1364 	 * It could be that they are running a long cleanup phase in the driver
1365 	 * e.g. MMU unmappings, or running other long teardown flow even before
1366 	 * our cleanup.
1367 	 * Therefore we need to wait again to make sure they are closed before
1368 	 * continuing with the reset.
1369 	 */
1370 
1371 wait_for_processes:
1372 	while ((!list_empty(fd_list)) && (pending_cnt)) {
1373 		dev_dbg(hdev->dev,
1374 			"Waiting for all unmap operations to finish before hard reset\n");
1375 
1376 		pending_cnt--;
1377 
1378 		ssleep(1);
1379 	}
1380 
1381 	/* All processes exited successfully */
1382 	if (list_empty(fd_list))
1383 		return 0;
1384 
1385 	/* Give up waiting for processes to exit */
1386 	if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS)
1387 		return -ETIME;
1388 
1389 	hdev->process_kill_trial_cnt++;
1390 
1391 	return -EBUSY;
1392 }
1393 
1394 static void device_disable_open_processes(struct hl_device *hdev, bool control_dev)
1395 {
1396 	struct list_head *fd_list;
1397 	struct hl_fpriv *hpriv;
1398 	struct mutex *fd_lock;
1399 
1400 	fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
1401 	fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
1402 
1403 	mutex_lock(fd_lock);
1404 	list_for_each_entry(hpriv, fd_list, dev_node)
1405 		hpriv->hdev = NULL;
1406 	mutex_unlock(fd_lock);
1407 }
1408 
1409 static void send_disable_pci_access(struct hl_device *hdev, u32 flags)
1410 {
1411 	/* If reset is due to heartbeat, device CPU is no responsive in
1412 	 * which case no point sending PCI disable message to it.
1413 	 */
1414 	if ((flags & HL_DRV_RESET_HARD) &&
1415 			!(flags & (HL_DRV_RESET_HEARTBEAT | HL_DRV_RESET_BYPASS_REQ_TO_FW))) {
1416 		/* Disable PCI access from device F/W so he won't send
1417 		 * us additional interrupts. We disable MSI/MSI-X at
1418 		 * the halt_engines function and we can't have the F/W
1419 		 * sending us interrupts after that. We need to disable
1420 		 * the access here because if the device is marked
1421 		 * disable, the message won't be send. Also, in case
1422 		 * of heartbeat, the device CPU is marked as disable
1423 		 * so this message won't be sent
1424 		 */
1425 		if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) {
1426 			dev_warn(hdev->dev, "Failed to disable FW's PCI access\n");
1427 			return;
1428 		}
1429 
1430 		/* verify that last EQs are handled before disabled is set */
1431 		if (hdev->cpu_queues_enable)
1432 			synchronize_irq(pci_irq_vector(hdev->pdev,
1433 					hdev->asic_prop.eq_interrupt_id));
1434 	}
1435 }
1436 
1437 static void handle_reset_trigger(struct hl_device *hdev, u32 flags)
1438 {
1439 	u32 cur_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
1440 
1441 	/* No consecutive mechanism when user context exists */
1442 	if (hdev->is_compute_ctx_active)
1443 		return;
1444 
1445 	/*
1446 	 * 'reset cause' is being updated here, because getting here
1447 	 * means that it's the 1st time and the last time we're here
1448 	 * ('in_reset' makes sure of it). This makes sure that
1449 	 * 'reset_cause' will continue holding its 1st recorded reason!
1450 	 */
1451 	if (flags & HL_DRV_RESET_HEARTBEAT) {
1452 		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT;
1453 		cur_reset_trigger = HL_DRV_RESET_HEARTBEAT;
1454 	} else if (flags & HL_DRV_RESET_TDR) {
1455 		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR;
1456 		cur_reset_trigger = HL_DRV_RESET_TDR;
1457 	} else if (flags & HL_DRV_RESET_FW_FATAL_ERR) {
1458 		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
1459 		cur_reset_trigger = HL_DRV_RESET_FW_FATAL_ERR;
1460 	} else {
1461 		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
1462 	}
1463 
1464 	/*
1465 	 * If reset cause is same twice, then reset_trigger_repeated
1466 	 * is set and if this reset is due to a fatal FW error
1467 	 * device is set to an unstable state.
1468 	 */
1469 	if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) {
1470 		hdev->reset_info.prev_reset_trigger = cur_reset_trigger;
1471 		hdev->reset_info.reset_trigger_repeated = 0;
1472 	} else {
1473 		hdev->reset_info.reset_trigger_repeated = 1;
1474 	}
1475 }
1476 
1477 /*
1478  * hl_device_reset - reset the device
1479  *
1480  * @hdev: pointer to habanalabs device structure
1481  * @flags: reset flags.
1482  *
1483  * Block future CS and wait for pending CS to be enqueued
1484  * Call ASIC H/W fini
1485  * Flush all completions
1486  * Re-initialize all internal data structures
1487  * Call ASIC H/W init, late_init
1488  * Test queues
1489  * Enable device
1490  *
1491  * Returns 0 for success or an error on failure.
1492  */
1493 int hl_device_reset(struct hl_device *hdev, u32 flags)
1494 {
1495 	bool hard_reset, from_hard_reset_thread, fw_reset, reset_upon_device_release,
1496 		schedule_hard_reset = false, delay_reset, from_dev_release, from_watchdog_thread;
1497 	u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
1498 	struct hl_ctx *ctx;
1499 	int i, rc, hw_fini_rc;
1500 
1501 	if (!hdev->init_done) {
1502 		dev_err(hdev->dev, "Can't reset before initialization is done\n");
1503 		return 0;
1504 	}
1505 
1506 	hard_reset = !!(flags & HL_DRV_RESET_HARD);
1507 	from_hard_reset_thread = !!(flags & HL_DRV_RESET_FROM_RESET_THR);
1508 	fw_reset = !!(flags & HL_DRV_RESET_BYPASS_REQ_TO_FW);
1509 	from_dev_release = !!(flags & HL_DRV_RESET_DEV_RELEASE);
1510 	delay_reset = !!(flags & HL_DRV_RESET_DELAY);
1511 	from_watchdog_thread = !!(flags & HL_DRV_RESET_FROM_WD_THR);
1512 	reset_upon_device_release = hdev->reset_upon_device_release && from_dev_release;
1513 
1514 	if (!hard_reset && (hl_device_status(hdev) == HL_DEVICE_STATUS_MALFUNCTION)) {
1515 		dev_dbg(hdev->dev, "soft-reset isn't supported on a malfunctioning device\n");
1516 		return 0;
1517 	}
1518 
1519 	if (!hard_reset && !hdev->asic_prop.supports_compute_reset) {
1520 		dev_dbg(hdev->dev, "asic doesn't support compute reset - do hard-reset instead\n");
1521 		hard_reset = true;
1522 	}
1523 
1524 	if (reset_upon_device_release) {
1525 		if (hard_reset) {
1526 			dev_crit(hdev->dev,
1527 				"Aborting reset because hard-reset is mutually exclusive with reset-on-device-release\n");
1528 			return -EINVAL;
1529 		}
1530 
1531 		goto do_reset;
1532 	}
1533 
1534 	if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) {
1535 		dev_dbg(hdev->dev,
1536 			"asic doesn't allow inference soft reset - do hard-reset instead\n");
1537 		hard_reset = true;
1538 	}
1539 
1540 do_reset:
1541 	/* Re-entry of reset thread */
1542 	if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
1543 		goto kill_processes;
1544 
1545 	/*
1546 	 * Prevent concurrency in this function - only one reset should be
1547 	 * done at any given time. We need to perform this only if we didn't
1548 	 * get here from a dedicated hard reset thread.
1549 	 */
1550 	if (!from_hard_reset_thread) {
1551 		/* Block future CS/VM/JOB completion operations */
1552 		spin_lock(&hdev->reset_info.lock);
1553 		if (hdev->reset_info.in_reset) {
1554 			/* We allow scheduling of a hard reset only during a compute reset */
1555 			if (hard_reset && hdev->reset_info.in_compute_reset)
1556 				hdev->reset_info.hard_reset_schedule_flags = flags;
1557 			spin_unlock(&hdev->reset_info.lock);
1558 			return 0;
1559 		}
1560 
1561 		/* This still allows the completion of some KDMA ops
1562 		 * Update this before in_reset because in_compute_reset implies we are in reset
1563 		 */
1564 		hdev->reset_info.in_compute_reset = !hard_reset;
1565 
1566 		hdev->reset_info.in_reset = 1;
1567 
1568 		spin_unlock(&hdev->reset_info.lock);
1569 
1570 		/* Cancel the device release watchdog work if required.
1571 		 * In case of reset-upon-device-release while the release watchdog work is
1572 		 * scheduled due to a hard-reset, do hard-reset instead of compute-reset.
1573 		 */
1574 		if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) {
1575 			struct hl_device_reset_work *watchdog_work =
1576 					&hdev->device_release_watchdog_work;
1577 
1578 			hdev->reset_info.watchdog_active = 0;
1579 			if (!from_watchdog_thread)
1580 				cancel_delayed_work_sync(&watchdog_work->reset_work);
1581 
1582 			if (from_dev_release && (watchdog_work->flags & HL_DRV_RESET_HARD)) {
1583 				hdev->reset_info.in_compute_reset = 0;
1584 				flags |= HL_DRV_RESET_HARD;
1585 				flags &= ~HL_DRV_RESET_DEV_RELEASE;
1586 				hard_reset = true;
1587 			}
1588 		}
1589 
1590 		if (delay_reset)
1591 			usleep_range(HL_RESET_DELAY_USEC, HL_RESET_DELAY_USEC << 1);
1592 
1593 escalate_reset_flow:
1594 		handle_reset_trigger(hdev, flags);
1595 		send_disable_pci_access(hdev, flags);
1596 
1597 		/* This also blocks future CS/VM/JOB completion operations */
1598 		hdev->disabled = true;
1599 
1600 		take_release_locks(hdev);
1601 
1602 		if (hard_reset)
1603 			dev_info(hdev->dev, "Going to reset device\n");
1604 		else if (reset_upon_device_release)
1605 			dev_dbg(hdev->dev, "Going to reset device after release by user\n");
1606 		else
1607 			dev_dbg(hdev->dev, "Going to reset engines of inference device\n");
1608 	}
1609 
1610 	if ((hard_reset) && (!from_hard_reset_thread)) {
1611 		hdev->reset_info.hard_reset_pending = true;
1612 
1613 		hdev->process_kill_trial_cnt = 0;
1614 
1615 		hdev->device_reset_work.flags = flags;
1616 
1617 		/*
1618 		 * Because the reset function can't run from heartbeat work,
1619 		 * we need to call the reset function from a dedicated work.
1620 		 */
1621 		queue_delayed_work(hdev->reset_wq, &hdev->device_reset_work.reset_work, 0);
1622 
1623 		return 0;
1624 	}
1625 
1626 	cleanup_resources(hdev, hard_reset, fw_reset, from_dev_release);
1627 
1628 kill_processes:
1629 	if (hard_reset) {
1630 		/* Kill processes here after CS rollback. This is because the
1631 		 * process can't really exit until all its CSs are done, which
1632 		 * is what we do in cs rollback
1633 		 */
1634 		rc = device_kill_open_processes(hdev, 0, false);
1635 
1636 		if (rc == -EBUSY) {
1637 			if (hdev->device_fini_pending) {
1638 				dev_crit(hdev->dev,
1639 					"%s Failed to kill all open processes, stopping hard reset\n",
1640 					dev_name(&(hdev)->pdev->dev));
1641 				goto out_err;
1642 			}
1643 
1644 			/* signal reset thread to reschedule */
1645 			return rc;
1646 		}
1647 
1648 		if (rc) {
1649 			dev_crit(hdev->dev,
1650 				"%s Failed to kill all open processes, stopping hard reset\n",
1651 				dev_name(&(hdev)->pdev->dev));
1652 			goto out_err;
1653 		}
1654 
1655 		/* Flush the Event queue workers to make sure no other thread is
1656 		 * reading or writing to registers during the reset
1657 		 */
1658 		flush_workqueue(hdev->eq_wq);
1659 	}
1660 
1661 	/* Reset the H/W. It will be in idle state after this returns */
1662 	hw_fini_rc = hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
1663 
1664 	if (hard_reset) {
1665 		hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
1666 
1667 		/* Release kernel context */
1668 		if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
1669 			hdev->kernel_ctx = NULL;
1670 
1671 		hl_vm_fini(hdev);
1672 		hl_mmu_fini(hdev);
1673 		hl_eq_reset(hdev, &hdev->event_queue);
1674 	}
1675 
1676 	/* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
1677 	hl_hw_queue_reset(hdev, hard_reset);
1678 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1679 		hl_cq_reset(hdev, &hdev->completion_queue[i]);
1680 
1681 	/* Make sure the context switch phase will run again */
1682 	ctx = hl_get_compute_ctx(hdev);
1683 	if (ctx) {
1684 		atomic_set(&ctx->thread_ctx_switch_token, 1);
1685 		ctx->thread_ctx_switch_wait_token = 0;
1686 		hl_ctx_put(ctx);
1687 	}
1688 
1689 	if (hw_fini_rc) {
1690 		rc = hw_fini_rc;
1691 		goto out_err;
1692 	}
1693 	/* Finished tear-down, starting to re-initialize */
1694 
1695 	if (hard_reset) {
1696 		hdev->device_cpu_disabled = false;
1697 		hdev->reset_info.hard_reset_pending = false;
1698 
1699 		if (hdev->reset_info.reset_trigger_repeated &&
1700 				(hdev->reset_info.prev_reset_trigger ==
1701 						HL_DRV_RESET_FW_FATAL_ERR)) {
1702 			/* if there 2 back to back resets from FW,
1703 			 * ensure driver puts the driver in a unusable state
1704 			 */
1705 			dev_crit(hdev->dev,
1706 				"%s Consecutive FW fatal errors received, stopping hard reset\n",
1707 				dev_name(&(hdev)->pdev->dev));
1708 			rc = -EIO;
1709 			goto out_err;
1710 		}
1711 
1712 		if (hdev->kernel_ctx) {
1713 			dev_crit(hdev->dev,
1714 				"%s kernel ctx was alive during hard reset, something is terribly wrong\n",
1715 				dev_name(&(hdev)->pdev->dev));
1716 			rc = -EBUSY;
1717 			goto out_err;
1718 		}
1719 
1720 		rc = hl_mmu_init(hdev);
1721 		if (rc) {
1722 			dev_err(hdev->dev,
1723 				"Failed to initialize MMU S/W after hard reset\n");
1724 			goto out_err;
1725 		}
1726 
1727 		/* Allocate the kernel context */
1728 		hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
1729 						GFP_KERNEL);
1730 		if (!hdev->kernel_ctx) {
1731 			rc = -ENOMEM;
1732 			hl_mmu_fini(hdev);
1733 			goto out_err;
1734 		}
1735 
1736 		hdev->is_compute_ctx_active = false;
1737 
1738 		rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1739 		if (rc) {
1740 			dev_err(hdev->dev,
1741 				"failed to init kernel ctx in hard reset\n");
1742 			kfree(hdev->kernel_ctx);
1743 			hdev->kernel_ctx = NULL;
1744 			hl_mmu_fini(hdev);
1745 			goto out_err;
1746 		}
1747 	}
1748 
1749 	/* Device is now enabled as part of the initialization requires
1750 	 * communication with the device firmware to get information that
1751 	 * is required for the initialization itself
1752 	 */
1753 	hdev->disabled = false;
1754 
1755 	/* F/W security enabled indication might be updated after hard-reset */
1756 	if (hard_reset) {
1757 		rc = hl_fw_read_preboot_status(hdev);
1758 		if (rc)
1759 			goto out_err;
1760 	}
1761 
1762 	rc = hdev->asic_funcs->hw_init(hdev);
1763 	if (rc) {
1764 		dev_err(hdev->dev, "failed to initialize the H/W after reset\n");
1765 		goto out_err;
1766 	}
1767 
1768 	/* If device is not idle fail the reset process */
1769 	if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
1770 						HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) {
1771 		print_idle_status_mask(hdev, "device is not idle after reset", idle_mask);
1772 		rc = -EIO;
1773 		goto out_err;
1774 	}
1775 
1776 	/* Check that the communication with the device is working */
1777 	rc = hdev->asic_funcs->test_queues(hdev);
1778 	if (rc) {
1779 		dev_err(hdev->dev, "Failed to detect if device is alive after reset\n");
1780 		goto out_err;
1781 	}
1782 
1783 	if (hard_reset) {
1784 		rc = device_late_init(hdev);
1785 		if (rc) {
1786 			dev_err(hdev->dev, "Failed late init after hard reset\n");
1787 			goto out_err;
1788 		}
1789 
1790 		rc = hl_vm_init(hdev);
1791 		if (rc) {
1792 			dev_err(hdev->dev, "Failed to init memory module after hard reset\n");
1793 			goto out_err;
1794 		}
1795 
1796 		if (!hdev->asic_prop.fw_security_enabled)
1797 			hl_fw_set_max_power(hdev);
1798 	} else {
1799 		rc = hdev->asic_funcs->compute_reset_late_init(hdev);
1800 		if (rc) {
1801 			if (reset_upon_device_release)
1802 				dev_err(hdev->dev,
1803 					"Failed late init in reset after device release\n");
1804 			else
1805 				dev_err(hdev->dev, "Failed late init after compute reset\n");
1806 			goto out_err;
1807 		}
1808 	}
1809 
1810 	rc = hdev->asic_funcs->scrub_device_mem(hdev);
1811 	if (rc) {
1812 		dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc);
1813 		goto out_err;
1814 	}
1815 
1816 	spin_lock(&hdev->reset_info.lock);
1817 	hdev->reset_info.in_compute_reset = 0;
1818 
1819 	/* Schedule hard reset only if requested and if not already in hard reset.
1820 	 * We keep 'in_reset' enabled, so no other reset can go in during the hard
1821 	 * reset schedule
1822 	 */
1823 	if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags)
1824 		schedule_hard_reset = true;
1825 	else
1826 		hdev->reset_info.in_reset = 0;
1827 
1828 	spin_unlock(&hdev->reset_info.lock);
1829 
1830 	hdev->reset_info.needs_reset = false;
1831 
1832 	if (hard_reset)
1833 		dev_info(hdev->dev,
1834 			 "Successfully finished resetting the %s device\n",
1835 			 dev_name(&(hdev)->pdev->dev));
1836 	else
1837 		dev_dbg(hdev->dev,
1838 			"Successfully finished resetting the %s device\n",
1839 			dev_name(&(hdev)->pdev->dev));
1840 
1841 	if (hard_reset) {
1842 		hdev->reset_info.hard_reset_cnt++;
1843 
1844 		/* After reset is done, we are ready to receive events from
1845 		 * the F/W. We can't do it before because we will ignore events
1846 		 * and if those events are fatal, we won't know about it and
1847 		 * the device will be operational although it shouldn't be
1848 		 */
1849 		hdev->asic_funcs->enable_events_from_fw(hdev);
1850 	} else {
1851 		if (!reset_upon_device_release)
1852 			hdev->reset_info.compute_reset_cnt++;
1853 
1854 		if (schedule_hard_reset) {
1855 			dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n");
1856 			flags = hdev->reset_info.hard_reset_schedule_flags;
1857 			hdev->reset_info.hard_reset_schedule_flags = 0;
1858 			hard_reset = true;
1859 			goto escalate_reset_flow;
1860 		}
1861 	}
1862 
1863 	return 0;
1864 
1865 out_err:
1866 	hdev->disabled = true;
1867 
1868 	spin_lock(&hdev->reset_info.lock);
1869 	hdev->reset_info.in_compute_reset = 0;
1870 
1871 	if (hard_reset) {
1872 		dev_err(hdev->dev,
1873 			"%s Failed to reset! Device is NOT usable\n",
1874 			dev_name(&(hdev)->pdev->dev));
1875 		hdev->reset_info.hard_reset_cnt++;
1876 	} else {
1877 		if (reset_upon_device_release) {
1878 			dev_err(hdev->dev, "Failed to reset device after user release\n");
1879 			flags &= ~HL_DRV_RESET_DEV_RELEASE;
1880 		} else {
1881 			dev_err(hdev->dev, "Failed to do compute reset\n");
1882 			hdev->reset_info.compute_reset_cnt++;
1883 		}
1884 
1885 		spin_unlock(&hdev->reset_info.lock);
1886 		flags |= HL_DRV_RESET_HARD;
1887 		hard_reset = true;
1888 		goto escalate_reset_flow;
1889 	}
1890 
1891 	hdev->reset_info.in_reset = 0;
1892 
1893 	spin_unlock(&hdev->reset_info.lock);
1894 
1895 	return rc;
1896 }
1897 
1898 /*
1899  * hl_device_cond_reset() - conditionally reset the device.
1900  * @hdev: pointer to habanalabs device structure.
1901  * @reset_flags: reset flags.
1902  * @event_mask: events to notify user about.
1903  *
1904  * Conditionally reset the device, or alternatively schedule a watchdog work to reset the device
1905  * unless another reset precedes it.
1906  */
1907 int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
1908 {
1909 	struct hl_ctx *ctx = NULL;
1910 
1911 	/* F/W reset cannot be postponed */
1912 	if (flags & HL_DRV_RESET_BYPASS_REQ_TO_FW)
1913 		goto device_reset;
1914 
1915 	/* Device release watchdog is relevant only if user exists and gets a reset notification */
1916 	if (!(event_mask & HL_NOTIFIER_EVENT_DEVICE_RESET)) {
1917 		dev_err(hdev->dev, "Resetting device without a reset indication to user\n");
1918 		goto device_reset;
1919 	}
1920 
1921 	ctx = hl_get_compute_ctx(hdev);
1922 	if (!ctx || !ctx->hpriv->notifier_event.eventfd)
1923 		goto device_reset;
1924 
1925 	/* Schedule the device release watchdog work unless reset is already in progress or if the
1926 	 * work is already scheduled.
1927 	 */
1928 	spin_lock(&hdev->reset_info.lock);
1929 	if (hdev->reset_info.in_reset) {
1930 		spin_unlock(&hdev->reset_info.lock);
1931 		goto device_reset;
1932 	}
1933 
1934 	if (hdev->reset_info.watchdog_active)
1935 		goto out;
1936 
1937 	hdev->device_release_watchdog_work.flags = flags;
1938 	dev_dbg(hdev->dev, "Device is going to be hard-reset in %u sec unless being released\n",
1939 		hdev->device_release_watchdog_timeout_sec);
1940 	schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work,
1941 				msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000));
1942 	hdev->reset_info.watchdog_active = 1;
1943 out:
1944 	spin_unlock(&hdev->reset_info.lock);
1945 
1946 	hl_notifier_event_send_all(hdev, event_mask);
1947 
1948 	hl_ctx_put(ctx);
1949 
1950 	hl_abort_waiting_for_completions(hdev);
1951 
1952 	return 0;
1953 
1954 device_reset:
1955 	if (event_mask)
1956 		hl_notifier_event_send_all(hdev, event_mask);
1957 	if (ctx)
1958 		hl_ctx_put(ctx);
1959 
1960 	return hl_device_reset(hdev, flags);
1961 }
1962 
1963 static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event_mask)
1964 {
1965 	mutex_lock(&notifier_event->lock);
1966 	notifier_event->events_mask |= event_mask;
1967 
1968 	if (notifier_event->eventfd)
1969 		eventfd_signal(notifier_event->eventfd, 1);
1970 
1971 	mutex_unlock(&notifier_event->lock);
1972 }
1973 
1974 /*
1975  * hl_notifier_event_send_all - notify all user processes via eventfd
1976  *
1977  * @hdev: pointer to habanalabs device structure
1978  * @event_mask: the occurred event/s
1979  * Returns 0 for success or an error on failure.
1980  */
1981 void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask)
1982 {
1983 	struct hl_fpriv	*hpriv;
1984 
1985 	if (!event_mask) {
1986 		dev_warn(hdev->dev, "Skip sending zero event");
1987 		return;
1988 	}
1989 
1990 	mutex_lock(&hdev->fpriv_list_lock);
1991 
1992 	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node)
1993 		hl_notifier_event_send(&hpriv->notifier_event, event_mask);
1994 
1995 	mutex_unlock(&hdev->fpriv_list_lock);
1996 
1997 	/* control device */
1998 	mutex_lock(&hdev->fpriv_ctrl_list_lock);
1999 
2000 	list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node)
2001 		hl_notifier_event_send(&hpriv->notifier_event, event_mask);
2002 
2003 	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
2004 }
2005 
2006 static int create_cdev(struct hl_device *hdev)
2007 {
2008 	char *name;
2009 	int rc;
2010 
2011 	hdev->cdev_idx = hdev->id / 2;
2012 
2013 	name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx);
2014 	if (!name) {
2015 		rc = -ENOMEM;
2016 		goto out_err;
2017 	}
2018 
2019 	/* Initialize cdev and device structures */
2020 	rc = device_init_cdev(hdev, hdev->hclass, hdev->id, &hl_ops, name,
2021 				&hdev->cdev, &hdev->dev);
2022 
2023 	kfree(name);
2024 
2025 	if (rc)
2026 		goto out_err;
2027 
2028 	name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx);
2029 	if (!name) {
2030 		rc = -ENOMEM;
2031 		goto free_dev;
2032 	}
2033 
2034 	/* Initialize cdev and device structures for control device */
2035 	rc = device_init_cdev(hdev, hdev->hclass, hdev->id_control, &hl_ctrl_ops,
2036 				name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
2037 
2038 	kfree(name);
2039 
2040 	if (rc)
2041 		goto free_dev;
2042 
2043 	return 0;
2044 
2045 free_dev:
2046 	put_device(hdev->dev);
2047 out_err:
2048 	return rc;
2049 }
2050 
2051 /*
2052  * hl_device_init - main initialization function for habanalabs device
2053  *
2054  * @hdev: pointer to habanalabs device structure
2055  *
2056  * Allocate an id for the device, do early initialization and then call the
2057  * ASIC specific initialization functions. Finally, create the cdev and the
2058  * Linux device to expose it to the user
2059  */
2060 int hl_device_init(struct hl_device *hdev)
2061 {
2062 	int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
2063 	bool expose_interfaces_on_err = false;
2064 
2065 	rc = create_cdev(hdev);
2066 	if (rc)
2067 		goto out_disabled;
2068 
2069 	/* Initialize ASIC function pointers and perform early init */
2070 	rc = device_early_init(hdev);
2071 	if (rc)
2072 		goto free_dev;
2073 
2074 	user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
2075 				hdev->asic_prop.user_interrupt_count;
2076 
2077 	if (user_interrupt_cnt) {
2078 		hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt),
2079 						GFP_KERNEL);
2080 		if (!hdev->user_interrupt) {
2081 			rc = -ENOMEM;
2082 			goto early_fini;
2083 		}
2084 	}
2085 
2086 	/*
2087 	 * Start calling ASIC initialization. First S/W then H/W and finally
2088 	 * late init
2089 	 */
2090 	rc = hdev->asic_funcs->sw_init(hdev);
2091 	if (rc)
2092 		goto free_usr_intr_mem;
2093 
2094 
2095 	/* initialize completion structure for multi CS wait */
2096 	hl_multi_cs_completion_init(hdev);
2097 
2098 	/*
2099 	 * Initialize the H/W queues. Must be done before hw_init, because
2100 	 * there the addresses of the kernel queue are being written to the
2101 	 * registers of the device
2102 	 */
2103 	rc = hl_hw_queues_create(hdev);
2104 	if (rc) {
2105 		dev_err(hdev->dev, "failed to initialize kernel queues\n");
2106 		goto sw_fini;
2107 	}
2108 
2109 	cq_cnt = hdev->asic_prop.completion_queues_count;
2110 
2111 	/*
2112 	 * Initialize the completion queues. Must be done before hw_init,
2113 	 * because there the addresses of the completion queues are being
2114 	 * passed as arguments to request_irq
2115 	 */
2116 	if (cq_cnt) {
2117 		hdev->completion_queue = kcalloc(cq_cnt,
2118 				sizeof(*hdev->completion_queue),
2119 				GFP_KERNEL);
2120 
2121 		if (!hdev->completion_queue) {
2122 			dev_err(hdev->dev,
2123 				"failed to allocate completion queues\n");
2124 			rc = -ENOMEM;
2125 			goto hw_queues_destroy;
2126 		}
2127 	}
2128 
2129 	for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
2130 		rc = hl_cq_init(hdev, &hdev->completion_queue[i],
2131 				hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
2132 		if (rc) {
2133 			dev_err(hdev->dev,
2134 				"failed to initialize completion queue\n");
2135 			goto cq_fini;
2136 		}
2137 		hdev->completion_queue[i].cq_idx = i;
2138 	}
2139 
2140 	hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs,
2141 					sizeof(struct hl_cs *), GFP_KERNEL);
2142 	if (!hdev->shadow_cs_queue) {
2143 		rc = -ENOMEM;
2144 		goto cq_fini;
2145 	}
2146 
2147 	/*
2148 	 * Initialize the event queue. Must be done before hw_init,
2149 	 * because there the address of the event queue is being
2150 	 * passed as argument to request_irq
2151 	 */
2152 	rc = hl_eq_init(hdev, &hdev->event_queue);
2153 	if (rc) {
2154 		dev_err(hdev->dev, "failed to initialize event queue\n");
2155 		goto free_shadow_cs_queue;
2156 	}
2157 
2158 	/* MMU S/W must be initialized before kernel context is created */
2159 	rc = hl_mmu_init(hdev);
2160 	if (rc) {
2161 		dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
2162 		goto eq_fini;
2163 	}
2164 
2165 	/* Allocate the kernel context */
2166 	hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
2167 	if (!hdev->kernel_ctx) {
2168 		rc = -ENOMEM;
2169 		goto mmu_fini;
2170 	}
2171 
2172 	hdev->is_compute_ctx_active = false;
2173 
2174 	hdev->asic_funcs->state_dump_init(hdev);
2175 
2176 	hdev->device_release_watchdog_timeout_sec = HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC;
2177 
2178 	hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL;
2179 
2180 	rc = hl_debugfs_device_init(hdev);
2181 	if (rc) {
2182 		dev_err(hdev->dev, "failed to initialize debugfs entry structure\n");
2183 		kfree(hdev->kernel_ctx);
2184 		goto mmu_fini;
2185 	}
2186 
2187 	/* The debugfs entry structure is accessed in hl_ctx_init(), so it must be called after
2188 	 * hl_debugfs_device_init().
2189 	 */
2190 	rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
2191 	if (rc) {
2192 		dev_err(hdev->dev, "failed to initialize kernel context\n");
2193 		kfree(hdev->kernel_ctx);
2194 		goto debugfs_device_fini;
2195 	}
2196 
2197 	rc = hl_cb_pool_init(hdev);
2198 	if (rc) {
2199 		dev_err(hdev->dev, "failed to initialize CB pool\n");
2200 		goto release_ctx;
2201 	}
2202 
2203 	rc = hl_dec_init(hdev);
2204 	if (rc) {
2205 		dev_err(hdev->dev, "Failed to initialize the decoder module\n");
2206 		goto cb_pool_fini;
2207 	}
2208 
2209 	/*
2210 	 * From this point, override rc (=0) in case of an error to allow debugging
2211 	 * (by adding char devices and creating sysfs/debugfs files as part of the error flow).
2212 	 */
2213 	expose_interfaces_on_err = true;
2214 
2215 	/* Device is now enabled as part of the initialization requires
2216 	 * communication with the device firmware to get information that
2217 	 * is required for the initialization itself
2218 	 */
2219 	hdev->disabled = false;
2220 
2221 	rc = hdev->asic_funcs->hw_init(hdev);
2222 	if (rc) {
2223 		dev_err(hdev->dev, "failed to initialize the H/W\n");
2224 		rc = 0;
2225 		goto out_disabled;
2226 	}
2227 
2228 	/* Check that the communication with the device is working */
2229 	rc = hdev->asic_funcs->test_queues(hdev);
2230 	if (rc) {
2231 		dev_err(hdev->dev, "Failed to detect if device is alive\n");
2232 		rc = 0;
2233 		goto out_disabled;
2234 	}
2235 
2236 	rc = device_late_init(hdev);
2237 	if (rc) {
2238 		dev_err(hdev->dev, "Failed late initialization\n");
2239 		rc = 0;
2240 		goto out_disabled;
2241 	}
2242 
2243 	dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
2244 		hdev->asic_name,
2245 		hdev->asic_prop.dram_size / SZ_1G);
2246 
2247 	rc = hl_vm_init(hdev);
2248 	if (rc) {
2249 		dev_err(hdev->dev, "Failed to initialize memory module\n");
2250 		rc = 0;
2251 		goto out_disabled;
2252 	}
2253 
2254 	/*
2255 	 * Expose devices and sysfs/debugfs files to user.
2256 	 * From here there is no need to expose them in case of an error.
2257 	 */
2258 	expose_interfaces_on_err = false;
2259 	rc = cdev_sysfs_debugfs_add(hdev);
2260 	if (rc) {
2261 		dev_err(hdev->dev, "Failed to add char devices and sysfs/debugfs files\n");
2262 		rc = 0;
2263 		goto out_disabled;
2264 	}
2265 
2266 	/* Need to call this again because the max power might change,
2267 	 * depending on card type for certain ASICs
2268 	 */
2269 	if (hdev->asic_prop.set_max_power_on_device_init &&
2270 			!hdev->asic_prop.fw_security_enabled)
2271 		hl_fw_set_max_power(hdev);
2272 
2273 	/*
2274 	 * hl_hwmon_init() must be called after device_late_init(), because only
2275 	 * there we get the information from the device about which
2276 	 * hwmon-related sensors the device supports.
2277 	 * Furthermore, it must be done after adding the device to the system.
2278 	 */
2279 	rc = hl_hwmon_init(hdev);
2280 	if (rc) {
2281 		dev_err(hdev->dev, "Failed to initialize hwmon\n");
2282 		rc = 0;
2283 		goto out_disabled;
2284 	}
2285 
2286 	dev_notice(hdev->dev,
2287 		"Successfully added device %s to habanalabs driver\n",
2288 		dev_name(&(hdev)->pdev->dev));
2289 
2290 	hdev->init_done = true;
2291 
2292 	/* After initialization is done, we are ready to receive events from
2293 	 * the F/W. We can't do it before because we will ignore events and if
2294 	 * those events are fatal, we won't know about it and the device will
2295 	 * be operational although it shouldn't be
2296 	 */
2297 	hdev->asic_funcs->enable_events_from_fw(hdev);
2298 
2299 	return 0;
2300 
2301 cb_pool_fini:
2302 	hl_cb_pool_fini(hdev);
2303 release_ctx:
2304 	if (hl_ctx_put(hdev->kernel_ctx) != 1)
2305 		dev_err(hdev->dev,
2306 			"kernel ctx is still alive on initialization failure\n");
2307 debugfs_device_fini:
2308 	hl_debugfs_device_fini(hdev);
2309 mmu_fini:
2310 	hl_mmu_fini(hdev);
2311 eq_fini:
2312 	hl_eq_fini(hdev, &hdev->event_queue);
2313 free_shadow_cs_queue:
2314 	kfree(hdev->shadow_cs_queue);
2315 cq_fini:
2316 	for (i = 0 ; i < cq_ready_cnt ; i++)
2317 		hl_cq_fini(hdev, &hdev->completion_queue[i]);
2318 	kfree(hdev->completion_queue);
2319 hw_queues_destroy:
2320 	hl_hw_queues_destroy(hdev);
2321 sw_fini:
2322 	hdev->asic_funcs->sw_fini(hdev);
2323 free_usr_intr_mem:
2324 	kfree(hdev->user_interrupt);
2325 early_fini:
2326 	device_early_fini(hdev);
2327 free_dev:
2328 	put_device(hdev->dev_ctrl);
2329 	put_device(hdev->dev);
2330 out_disabled:
2331 	hdev->disabled = true;
2332 	if (expose_interfaces_on_err)
2333 		cdev_sysfs_debugfs_add(hdev);
2334 	dev_err(&hdev->pdev->dev,
2335 		"Failed to initialize hl%d. Device %s is NOT usable !\n",
2336 		hdev->cdev_idx, dev_name(&hdev->pdev->dev));
2337 
2338 	return rc;
2339 }
2340 
2341 /*
2342  * hl_device_fini - main tear-down function for habanalabs device
2343  *
2344  * @hdev: pointer to habanalabs device structure
2345  *
2346  * Destroy the device, call ASIC fini functions and release the id
2347  */
2348 void hl_device_fini(struct hl_device *hdev)
2349 {
2350 	bool device_in_reset;
2351 	ktime_t timeout;
2352 	u64 reset_sec;
2353 	int i, rc;
2354 
2355 	dev_info(hdev->dev, "Removing device\n");
2356 
2357 	hdev->device_fini_pending = 1;
2358 	flush_delayed_work(&hdev->device_reset_work.reset_work);
2359 
2360 	if (hdev->pldm)
2361 		reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT;
2362 	else
2363 		reset_sec = HL_HARD_RESET_MAX_TIMEOUT;
2364 
2365 	/*
2366 	 * This function is competing with the reset function, so try to
2367 	 * take the reset atomic and if we are already in middle of reset,
2368 	 * wait until reset function is finished. Reset function is designed
2369 	 * to always finish. However, in Gaudi, because of all the network
2370 	 * ports, the hard reset could take between 10-30 seconds
2371 	 */
2372 
2373 	timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000);
2374 
2375 	spin_lock(&hdev->reset_info.lock);
2376 	device_in_reset = !!hdev->reset_info.in_reset;
2377 	if (!device_in_reset)
2378 		hdev->reset_info.in_reset = 1;
2379 	spin_unlock(&hdev->reset_info.lock);
2380 
2381 	while (device_in_reset) {
2382 		usleep_range(50, 200);
2383 
2384 		spin_lock(&hdev->reset_info.lock);
2385 		device_in_reset = !!hdev->reset_info.in_reset;
2386 		if (!device_in_reset)
2387 			hdev->reset_info.in_reset = 1;
2388 		spin_unlock(&hdev->reset_info.lock);
2389 
2390 		if (ktime_compare(ktime_get(), timeout) > 0) {
2391 			dev_crit(hdev->dev,
2392 				"%s Failed to remove device because reset function did not finish\n",
2393 				dev_name(&(hdev)->pdev->dev));
2394 			return;
2395 		}
2396 	}
2397 
2398 	cancel_delayed_work_sync(&hdev->device_release_watchdog_work.reset_work);
2399 
2400 	/* Disable PCI access from device F/W so it won't send us additional
2401 	 * interrupts. We disable MSI/MSI-X at the halt_engines function and we
2402 	 * can't have the F/W sending us interrupts after that. We need to
2403 	 * disable the access here because if the device is marked disable, the
2404 	 * message won't be send. Also, in case of heartbeat, the device CPU is
2405 	 * marked as disable so this message won't be sent
2406 	 */
2407 	hl_fw_send_pci_access_msg(hdev,	CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
2408 
2409 	/* Mark device as disabled */
2410 	hdev->disabled = true;
2411 
2412 	take_release_locks(hdev);
2413 
2414 	hdev->reset_info.hard_reset_pending = true;
2415 
2416 	hl_hwmon_fini(hdev);
2417 
2418 	cleanup_resources(hdev, true, false, false);
2419 
2420 	/* Kill processes here after CS rollback. This is because the process
2421 	 * can't really exit until all its CSs are done, which is what we
2422 	 * do in cs rollback
2423 	 */
2424 	dev_info(hdev->dev,
2425 		"Waiting for all processes to exit (timeout of %u seconds)",
2426 		HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI);
2427 
2428 	hdev->process_kill_trial_cnt = 0;
2429 	rc = device_kill_open_processes(hdev, HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI, false);
2430 	if (rc) {
2431 		dev_crit(hdev->dev, "Failed to kill all open processes\n");
2432 		device_disable_open_processes(hdev, false);
2433 	}
2434 
2435 	hdev->process_kill_trial_cnt = 0;
2436 	rc = device_kill_open_processes(hdev, 0, true);
2437 	if (rc) {
2438 		dev_crit(hdev->dev, "Failed to kill all control device open processes\n");
2439 		device_disable_open_processes(hdev, true);
2440 	}
2441 
2442 	hl_cb_pool_fini(hdev);
2443 
2444 	/* Reset the H/W. It will be in idle state after this returns */
2445 	rc = hdev->asic_funcs->hw_fini(hdev, true, false);
2446 	if (rc)
2447 		dev_err(hdev->dev, "hw_fini failed in device fini while removing device %d\n", rc);
2448 
2449 	hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
2450 
2451 	/* Release kernel context */
2452 	if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
2453 		dev_err(hdev->dev, "kernel ctx is still alive\n");
2454 
2455 	hl_dec_fini(hdev);
2456 
2457 	hl_vm_fini(hdev);
2458 
2459 	hl_mmu_fini(hdev);
2460 
2461 	vfree(hdev->captured_err_info.page_fault_info.user_mappings);
2462 
2463 	hl_eq_fini(hdev, &hdev->event_queue);
2464 
2465 	kfree(hdev->shadow_cs_queue);
2466 
2467 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
2468 		hl_cq_fini(hdev, &hdev->completion_queue[i]);
2469 	kfree(hdev->completion_queue);
2470 	kfree(hdev->user_interrupt);
2471 
2472 	hl_hw_queues_destroy(hdev);
2473 
2474 	/* Call ASIC S/W finalize function */
2475 	hdev->asic_funcs->sw_fini(hdev);
2476 
2477 	device_early_fini(hdev);
2478 
2479 	/* Hide devices and sysfs/debugfs files from user */
2480 	cdev_sysfs_debugfs_remove(hdev);
2481 
2482 	hl_debugfs_device_fini(hdev);
2483 
2484 	pr_info("removed device successfully\n");
2485 }
2486 
2487 /*
2488  * MMIO register access helper functions.
2489  */
2490 
2491 /*
2492  * hl_rreg - Read an MMIO register
2493  *
2494  * @hdev: pointer to habanalabs device structure
2495  * @reg: MMIO register offset (in bytes)
2496  *
2497  * Returns the value of the MMIO register we are asked to read
2498  *
2499  */
2500 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
2501 {
2502 	u32 val = readl(hdev->rmmio + reg);
2503 
2504 	if (unlikely(trace_habanalabs_rreg32_enabled()))
2505 		trace_habanalabs_rreg32(hdev->dev, reg, val);
2506 
2507 	return val;
2508 }
2509 
2510 /*
2511  * hl_wreg - Write to an MMIO register
2512  *
2513  * @hdev: pointer to habanalabs device structure
2514  * @reg: MMIO register offset (in bytes)
2515  * @val: 32-bit value
2516  *
2517  * Writes the 32-bit value into the MMIO register
2518  *
2519  */
2520 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
2521 {
2522 	if (unlikely(trace_habanalabs_wreg32_enabled()))
2523 		trace_habanalabs_wreg32(hdev->dev, reg, val);
2524 
2525 	writel(val, hdev->rmmio + reg);
2526 }
2527 
2528 void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
2529 			u8 flags)
2530 {
2531 	struct razwi_info *razwi_info = &hdev->captured_err_info.razwi_info;
2532 
2533 	if (num_of_engines > HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR) {
2534 		dev_err(hdev->dev,
2535 				"Number of possible razwi initiators (%u) exceeded limit (%u)\n",
2536 				num_of_engines, HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR);
2537 		return;
2538 	}
2539 
2540 	/* In case it's the first razwi since the device was opened, capture its parameters */
2541 	if (atomic_cmpxchg(&hdev->captured_err_info.razwi_info.razwi_detected, 0, 1))
2542 		return;
2543 
2544 	razwi_info->razwi.timestamp = ktime_to_ns(ktime_get());
2545 	razwi_info->razwi.addr = addr;
2546 	razwi_info->razwi.num_of_possible_engines = num_of_engines;
2547 	memcpy(&razwi_info->razwi.engine_id[0], &engine_id[0],
2548 			num_of_engines * sizeof(u16));
2549 	razwi_info->razwi.flags = flags;
2550 
2551 	razwi_info->razwi_info_available = true;
2552 }
2553 
2554 void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
2555 			u8 flags, u64 *event_mask)
2556 {
2557 	hl_capture_razwi(hdev, addr, engine_id, num_of_engines, flags);
2558 
2559 	if (event_mask)
2560 		*event_mask |= HL_NOTIFIER_EVENT_RAZWI;
2561 }
2562 
2563 static void hl_capture_user_mappings(struct hl_device *hdev, bool is_pmmu)
2564 {
2565 	struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info;
2566 	struct hl_vm_phys_pg_pack *phys_pg_pack = NULL;
2567 	struct hl_vm_hash_node *hnode;
2568 	struct hl_userptr *userptr;
2569 	enum vm_type *vm_type;
2570 	struct hl_ctx *ctx;
2571 	u32 map_idx = 0;
2572 	int i;
2573 
2574 	/* Reset previous session count*/
2575 	pgf_info->num_of_user_mappings = 0;
2576 
2577 	ctx = hl_get_compute_ctx(hdev);
2578 	if (!ctx) {
2579 		dev_err(hdev->dev, "Can't get user context for user mappings\n");
2580 		return;
2581 	}
2582 
2583 	mutex_lock(&ctx->mem_hash_lock);
2584 	hash_for_each(ctx->mem_hash, i, hnode, node) {
2585 		vm_type = hnode->ptr;
2586 		if (((*vm_type == VM_TYPE_USERPTR) && is_pmmu) ||
2587 				((*vm_type == VM_TYPE_PHYS_PACK) && !is_pmmu))
2588 			pgf_info->num_of_user_mappings++;
2589 
2590 	}
2591 
2592 	if (!pgf_info->num_of_user_mappings)
2593 		goto finish;
2594 
2595 	/* In case we already allocated in previous session, need to release it before
2596 	 * allocating new buffer.
2597 	 */
2598 	vfree(pgf_info->user_mappings);
2599 	pgf_info->user_mappings =
2600 			vzalloc(pgf_info->num_of_user_mappings * sizeof(struct hl_user_mapping));
2601 	if (!pgf_info->user_mappings) {
2602 		pgf_info->num_of_user_mappings = 0;
2603 		goto finish;
2604 	}
2605 
2606 	hash_for_each(ctx->mem_hash, i, hnode, node) {
2607 		vm_type = hnode->ptr;
2608 		if ((*vm_type == VM_TYPE_USERPTR) && (is_pmmu)) {
2609 			userptr = hnode->ptr;
2610 			pgf_info->user_mappings[map_idx].dev_va = hnode->vaddr;
2611 			pgf_info->user_mappings[map_idx].size = userptr->size;
2612 			map_idx++;
2613 		} else if ((*vm_type == VM_TYPE_PHYS_PACK) && (!is_pmmu)) {
2614 			phys_pg_pack = hnode->ptr;
2615 			pgf_info->user_mappings[map_idx].dev_va = hnode->vaddr;
2616 			pgf_info->user_mappings[map_idx].size = phys_pg_pack->total_size;
2617 			map_idx++;
2618 		}
2619 	}
2620 finish:
2621 	mutex_unlock(&ctx->mem_hash_lock);
2622 	hl_ctx_put(ctx);
2623 }
2624 
2625 void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu)
2626 {
2627 	struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info;
2628 
2629 	/* Capture only the first page fault */
2630 	if (atomic_cmpxchg(&pgf_info->page_fault_detected, 0, 1))
2631 		return;
2632 
2633 	pgf_info->page_fault.timestamp = ktime_to_ns(ktime_get());
2634 	pgf_info->page_fault.addr = addr;
2635 	pgf_info->page_fault.engine_id = eng_id;
2636 	hl_capture_user_mappings(hdev, is_pmmu);
2637 
2638 	pgf_info->page_fault_info_available = true;
2639 }
2640 
2641 void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu,
2642 				u64 *event_mask)
2643 {
2644 	hl_capture_page_fault(hdev, addr, eng_id, is_pmmu);
2645 
2646 	if (event_mask)
2647 		*event_mask |=  HL_NOTIFIER_EVENT_PAGE_FAULT;
2648 }
2649 
2650 static void hl_capture_hw_err(struct hl_device *hdev, u16 event_id)
2651 {
2652 	struct hw_err_info *info = &hdev->captured_err_info.hw_err;
2653 
2654 	/* Capture only the first HW err */
2655 	if (atomic_cmpxchg(&info->event_detected, 0, 1))
2656 		return;
2657 
2658 	info->event.timestamp = ktime_to_ns(ktime_get());
2659 	info->event.event_id = event_id;
2660 
2661 	info->event_info_available = true;
2662 }
2663 
2664 void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask)
2665 {
2666 	hl_capture_hw_err(hdev, event_id);
2667 
2668 	if (event_mask)
2669 		*event_mask |= HL_NOTIFIER_EVENT_CRITICL_HW_ERR;
2670 }
2671 
2672 static void hl_capture_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *fw_info)
2673 {
2674 	struct fw_err_info *info = &hdev->captured_err_info.fw_err;
2675 
2676 	/* Capture only the first FW error */
2677 	if (atomic_cmpxchg(&info->event_detected, 0, 1))
2678 		return;
2679 
2680 	info->event.timestamp = ktime_to_ns(ktime_get());
2681 	info->event.err_type = fw_info->err_type;
2682 	if (fw_info->err_type == HL_INFO_FW_REPORTED_ERR)
2683 		info->event.event_id = fw_info->event_id;
2684 
2685 	info->event_info_available = true;
2686 }
2687 
2688 void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info)
2689 {
2690 	hl_capture_fw_err(hdev, info);
2691 
2692 	if (info->event_mask)
2693 		*info->event_mask |= HL_NOTIFIER_EVENT_CRITICL_FW_ERR;
2694 }
2695 
2696 void hl_enable_err_info_capture(struct hl_error_info *captured_err_info)
2697 {
2698 	vfree(captured_err_info->page_fault_info.user_mappings);
2699 	memset(captured_err_info, 0, sizeof(struct hl_error_info));
2700 	atomic_set(&captured_err_info->cs_timeout.write_enable, 1);
2701 	captured_err_info->undef_opcode.write_enable = true;
2702 }
2703