1 /* SPDX-License-Identifier: GPL-2.0
2  *
3  * Copyright 2019-2022 HabanaLabs, Ltd.
4  * All Rights Reserved.
5  *
6  */
7 
8 #ifndef GAUDIP_H_
9 #define GAUDIP_H_
10 
11 #include <uapi/drm/habanalabs_accel.h>
12 #include "../common/habanalabs.h"
13 #include "../include/common/hl_boot_if.h"
14 #include "../include/gaudi/gaudi_packets.h"
15 #include "../include/gaudi/gaudi.h"
16 #include "../include/gaudi/gaudi_async_events.h"
17 #include "../include/gaudi/gaudi_fw_if.h"
18 
19 #define NUMBER_OF_EXT_HW_QUEUES		8
20 #define NUMBER_OF_CMPLT_QUEUES		NUMBER_OF_EXT_HW_QUEUES
21 #define NUMBER_OF_CPU_HW_QUEUES		1
22 #define NUMBER_OF_INT_HW_QUEUES		100
23 #define NUMBER_OF_HW_QUEUES		(NUMBER_OF_EXT_HW_QUEUES + \
24 					NUMBER_OF_CPU_HW_QUEUES + \
25 					NUMBER_OF_INT_HW_QUEUES)
26 
27 /* 10 NIC QMANs, DMA5 QMAN, TPC7 QMAN */
28 #define NUMBER_OF_COLLECTIVE_QUEUES	12
29 #define NUMBER_OF_SOBS_IN_GRP		11
30 
31 #define GAUDI_STREAM_MASTER_ARR_SIZE	8
32 
33 #define CORESIGHT_TIMEOUT_USEC		100000		/* 100 ms */
34 
35 #define GAUDI_MAX_CLK_FREQ		2200000000ull	/* 2200 MHz */
36 
37 #define MAX_POWER_DEFAULT_PCI		200000		/* 200W */
38 #define MAX_POWER_DEFAULT_PMC		350000		/* 350W */
39 
40 #define DC_POWER_DEFAULT_PCI		60000		/* 60W */
41 #define DC_POWER_DEFAULT_PMC		60000		/* 60W */
42 
43 #define DC_POWER_DEFAULT_PMC_SEC	97000		/* 97W */
44 
45 #define GAUDI_CPU_TIMEOUT_USEC		30000000	/* 30s */
46 
47 #define TPC_ENABLED_MASK		0xFF
48 
49 #define GAUDI_HBM_SIZE_32GB		0x800000000ull
50 #define GAUDI_HBM_DEVICES		4
51 #define GAUDI_HBM_CHANNELS		8
52 #define GAUDI_HBM_CFG_BASE		(mmHBM0_BASE - CFG_BASE)
53 #define GAUDI_HBM_CFG_OFFSET		(mmHBM1_BASE - mmHBM0_BASE)
54 
55 #define DMA_MAX_TRANSFER_SIZE		U32_MAX
56 
57 #define GAUDI_DEFAULT_CARD_NAME		"HL205"
58 
59 #define GAUDI_MAX_PENDING_CS		SZ_16K
60 
61 #if !IS_MAX_PENDING_CS_VALID(GAUDI_MAX_PENDING_CS)
62 #error "GAUDI_MAX_PENDING_CS must be power of 2 and greater than 1"
63 #endif
64 
65 #define PCI_DMA_NUMBER_OF_CHNLS		2
66 #define HBM_DMA_NUMBER_OF_CHNLS		6
67 #define DMA_NUMBER_OF_CHNLS		(PCI_DMA_NUMBER_OF_CHNLS + \
68 						HBM_DMA_NUMBER_OF_CHNLS)
69 
70 #define MME_NUMBER_OF_SLAVE_ENGINES	2
71 #define MME_NUMBER_OF_ENGINES		(MME_NUMBER_OF_MASTER_ENGINES + \
72 					MME_NUMBER_OF_SLAVE_ENGINES)
73 #define MME_NUMBER_OF_QMANS		(MME_NUMBER_OF_MASTER_ENGINES * \
74 					QMAN_STREAMS)
75 
76 #define QMAN_STREAMS		4
77 #define PQ_FETCHER_CACHE_SIZE	8
78 
79 #define DMA_QMAN_OFFSET		(mmDMA1_QM_BASE - mmDMA0_QM_BASE)
80 #define TPC_QMAN_OFFSET		(mmTPC1_QM_BASE - mmTPC0_QM_BASE)
81 #define MME_QMAN_OFFSET		(mmMME1_QM_BASE - mmMME0_QM_BASE)
82 #define NIC_MACRO_QMAN_OFFSET	(mmNIC1_QM0_BASE - mmNIC0_QM0_BASE)
83 #define NIC_ENGINE_QMAN_OFFSET	(mmNIC0_QM1_BASE - mmNIC0_QM0_BASE)
84 
85 #define TPC_CFG_OFFSET		(mmTPC1_CFG_BASE - mmTPC0_CFG_BASE)
86 
87 #define DMA_CORE_OFFSET		(mmDMA1_CORE_BASE - mmDMA0_CORE_BASE)
88 
89 #define QMAN_LDMA_SRC_OFFSET	(mmDMA0_CORE_SRC_BASE_LO - mmDMA0_CORE_CFG_0)
90 #define QMAN_LDMA_DST_OFFSET	(mmDMA0_CORE_DST_BASE_LO - mmDMA0_CORE_CFG_0)
91 #define QMAN_LDMA_SIZE_OFFSET	(mmDMA0_CORE_DST_TSIZE_0 - mmDMA0_CORE_CFG_0)
92 
93 #define QMAN_CPDMA_SRC_OFFSET	(mmDMA0_QM_CQ_PTR_LO_4 - mmDMA0_CORE_CFG_0)
94 #define QMAN_CPDMA_DST_OFFSET	(mmDMA0_CORE_DST_BASE_LO - mmDMA0_CORE_CFG_0)
95 #define QMAN_CPDMA_SIZE_OFFSET	(mmDMA0_QM_CQ_TSIZE_4 - mmDMA0_CORE_CFG_0)
96 
97 #define SIF_RTR_CTRL_OFFSET	(mmSIF_RTR_CTRL_1_BASE - mmSIF_RTR_CTRL_0_BASE)
98 
99 #define NIF_RTR_CTRL_OFFSET	(mmNIF_RTR_CTRL_1_BASE - mmNIF_RTR_CTRL_0_BASE)
100 
101 #define MME_ACC_OFFSET		(mmMME1_ACC_BASE - mmMME0_ACC_BASE)
102 #define SRAM_BANK_OFFSET	(mmSRAM_Y0_X1_RTR_BASE - mmSRAM_Y0_X0_RTR_BASE)
103 
104 #define NUM_OF_SOB_IN_BLOCK		\
105 	(((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 - \
106 	mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0) + 4) >> 2)
107 
108 #define NUM_OF_MONITORS_IN_BLOCK	\
109 	(((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 - \
110 	mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0) + 4) >> 2)
111 
112 #define MONITOR_MAX_SOBS	8
113 
114 /* DRAM Memory Map */
115 
116 #define CPU_FW_IMAGE_SIZE	0x10000000	/* 256MB */
117 #define MMU_PAGE_TABLES_SIZE	0x0BF00000	/* 191MB */
118 #define MMU_CACHE_MNG_SIZE	0x00100000	/* 1MB */
119 #define RESERVED		0x04000000	/* 64MB */
120 
121 #define CPU_FW_IMAGE_ADDR	DRAM_PHYS_BASE
122 #define MMU_PAGE_TABLES_ADDR	(CPU_FW_IMAGE_ADDR + CPU_FW_IMAGE_SIZE)
123 #define MMU_CACHE_MNG_ADDR	(MMU_PAGE_TABLES_ADDR + MMU_PAGE_TABLES_SIZE)
124 
125 #define DRAM_DRIVER_END_ADDR	(MMU_CACHE_MNG_ADDR + MMU_CACHE_MNG_SIZE +\
126 								RESERVED)
127 
128 #define DRAM_BASE_ADDR_USER	0x20000000
129 
130 #if (DRAM_DRIVER_END_ADDR > DRAM_BASE_ADDR_USER)
131 #error "Driver must reserve no more than 512MB"
132 #endif
133 
134 /* Internal QMANs PQ sizes */
135 
136 #define MME_QMAN_LENGTH			1024
137 #define MME_QMAN_SIZE_IN_BYTES		(MME_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
138 
139 #define HBM_DMA_QMAN_LENGTH		4096
140 #define HBM_DMA_QMAN_SIZE_IN_BYTES	\
141 				(HBM_DMA_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
142 
143 #define TPC_QMAN_LENGTH			1024
144 #define TPC_QMAN_SIZE_IN_BYTES		(TPC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
145 
146 #define NIC_QMAN_LENGTH			4096
147 #define NIC_QMAN_SIZE_IN_BYTES		(NIC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
148 
149 
150 #define SRAM_USER_BASE_OFFSET  GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START
151 
152 /* Virtual address space */
153 #define VA_HOST_SPACE_START	0x1000000000000ull	/* 256TB */
154 #define VA_HOST_SPACE_END	0x3FF8000000000ull	/* 1PB - 512GB */
155 #define VA_HOST_SPACE_SIZE	(VA_HOST_SPACE_END - \
156 					VA_HOST_SPACE_START) /* 767TB */
157 #define HOST_SPACE_INTERNAL_CB_SZ	SZ_2M
158 
159 #define HW_CAP_PLL		BIT(0)
160 #define HW_CAP_HBM		BIT(1)
161 #define HW_CAP_MMU		BIT(2)
162 #define HW_CAP_MME		BIT(3)
163 #define HW_CAP_CPU		BIT(4)
164 #define HW_CAP_PCI_DMA		BIT(5)
165 #define HW_CAP_MSI		BIT(6)
166 #define HW_CAP_CPU_Q		BIT(7)
167 #define HW_CAP_HBM_DMA		BIT(8)
168 #define HW_CAP_SRAM_SCRAMBLER	BIT(10)
169 #define HW_CAP_HBM_SCRAMBLER	BIT(11)
170 
171 #define HW_CAP_NIC0		BIT(14)
172 #define HW_CAP_NIC1		BIT(15)
173 #define HW_CAP_NIC2		BIT(16)
174 #define HW_CAP_NIC3		BIT(17)
175 #define HW_CAP_NIC4		BIT(18)
176 #define HW_CAP_NIC5		BIT(19)
177 #define HW_CAP_NIC6		BIT(20)
178 #define HW_CAP_NIC7		BIT(21)
179 #define HW_CAP_NIC8		BIT(22)
180 #define HW_CAP_NIC9		BIT(23)
181 #define HW_CAP_NIC_MASK		GENMASK(23, 14)
182 #define HW_CAP_NIC_SHIFT	14
183 
184 #define HW_CAP_TPC0		BIT(24)
185 #define HW_CAP_TPC1		BIT(25)
186 #define HW_CAP_TPC2		BIT(26)
187 #define HW_CAP_TPC3		BIT(27)
188 #define HW_CAP_TPC4		BIT(28)
189 #define HW_CAP_TPC5		BIT(29)
190 #define HW_CAP_TPC6		BIT(30)
191 #define HW_CAP_TPC7		BIT(31)
192 #define HW_CAP_TPC_MASK		GENMASK(31, 24)
193 #define HW_CAP_TPC_SHIFT	24
194 
195 #define NEXT_SYNC_OBJ_ADDR_INTERVAL \
196 	(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 - \
197 	 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0)
198 #define NUM_OF_MME_ENGINES			2
199 #define NUM_OF_MME_SUB_ENGINES		2
200 #define NUM_OF_TPC_ENGINES			8
201 #define NUM_OF_DMA_ENGINES			8
202 #define NUM_OF_QUEUES				5
203 #define NUM_OF_STREAMS				4
204 #define NUM_OF_FENCES				4
205 
206 
207 #define GAUDI_CPU_PCI_MSB_ADDR(addr)	(((addr) & GENMASK_ULL(49, 39)) >> 39)
208 #define GAUDI_PCI_TO_CPU_ADDR(addr)			\
209 	do {						\
210 		(addr) &= ~GENMASK_ULL(49, 39);		\
211 		(addr) |= BIT_ULL(39);			\
212 	} while (0)
213 #define GAUDI_CPU_TO_PCI_ADDR(addr, extension)		\
214 	do {						\
215 		(addr) &= ~GENMASK_ULL(49, 39);		\
216 		(addr) |= (u64) (extension) << 39;	\
217 	} while (0)
218 
219 enum gaudi_dma_channels {
220 	GAUDI_PCI_DMA_1,
221 	GAUDI_PCI_DMA_2,
222 	GAUDI_HBM_DMA_1,
223 	GAUDI_HBM_DMA_2,
224 	GAUDI_HBM_DMA_3,
225 	GAUDI_HBM_DMA_4,
226 	GAUDI_HBM_DMA_5,
227 	GAUDI_HBM_DMA_6,
228 	GAUDI_DMA_MAX
229 };
230 
231 enum gaudi_tpc_mask {
232 	GAUDI_TPC_MASK_TPC0 = 0x01,
233 	GAUDI_TPC_MASK_TPC1 = 0x02,
234 	GAUDI_TPC_MASK_TPC2 = 0x04,
235 	GAUDI_TPC_MASK_TPC3 = 0x08,
236 	GAUDI_TPC_MASK_TPC4 = 0x10,
237 	GAUDI_TPC_MASK_TPC5 = 0x20,
238 	GAUDI_TPC_MASK_TPC6 = 0x40,
239 	GAUDI_TPC_MASK_TPC7 = 0x80,
240 	GAUDI_TPC_MASK_ALL = 0xFF
241 };
242 
243 enum gaudi_nic_mask {
244 	GAUDI_NIC_MASK_NIC0 = 0x01,
245 	GAUDI_NIC_MASK_NIC1 = 0x02,
246 	GAUDI_NIC_MASK_NIC2 = 0x04,
247 	GAUDI_NIC_MASK_NIC3 = 0x08,
248 	GAUDI_NIC_MASK_NIC4 = 0x10,
249 	GAUDI_NIC_MASK_NIC5 = 0x20,
250 	GAUDI_NIC_MASK_NIC6 = 0x40,
251 	GAUDI_NIC_MASK_NIC7 = 0x80,
252 	GAUDI_NIC_MASK_NIC8 = 0x100,
253 	GAUDI_NIC_MASK_NIC9 = 0x200,
254 	GAUDI_NIC_MASK_ALL = 0x3FF
255 };
256 
257 /*
258  * struct gaudi_hw_sob_group - H/W SOB group info.
259  * @hdev: habanalabs device structure.
260  * @kref: refcount of this SOB group. group will reset once refcount is zero.
261  * @base_sob_id: base sob id of this SOB group.
262  * @queue_id: id of the queue that waits on this sob group
263  */
264 struct gaudi_hw_sob_group {
265 	struct hl_device	*hdev;
266 	struct kref		kref;
267 	u32			base_sob_id;
268 	u32			queue_id;
269 };
270 
271 #define NUM_SOB_GROUPS (HL_RSVD_SOBS * QMAN_STREAMS)
272 /**
273  * struct gaudi_collective_properties -
274  *     holds all SOB groups and queues info reserved for the collective
275  * @hw_sob_group: H/W SOB groups.
276  * @next_sob_group_val: the next value to use for the currently used SOB group.
277  * @curr_sob_group_idx: the index of the currently used SOB group.
278  * @mstr_sob_mask: pre-defined masks for collective master monitors
279  */
280 struct gaudi_collective_properties {
281 	struct gaudi_hw_sob_group hw_sob_group[NUM_SOB_GROUPS];
282 	u16			next_sob_group_val[QMAN_STREAMS];
283 	u8			curr_sob_group_idx[QMAN_STREAMS];
284 	u8			mstr_sob_mask[HL_COLLECTIVE_RSVD_MSTR_MONS];
285 };
286 
287 /**
288  * struct gaudi_internal_qman_info - Internal QMAN information.
289  * @pq_kernel_addr: Kernel address of the PQ memory area in the host.
290  * @pq_dma_addr: DMA address of the PQ memory area in the host.
291  * @pq_size: Size of allocated host memory for PQ.
292  */
293 struct gaudi_internal_qman_info {
294 	void		*pq_kernel_addr;
295 	dma_addr_t	pq_dma_addr;
296 	size_t		pq_size;
297 };
298 
299 /**
300  * struct gaudi_device - ASIC specific manage structure.
301  * @cpucp_info_get: get information on device from CPU-CP
302  * @hw_queues_lock: protects the H/W queues from concurrent access.
303  * @internal_qmans: Internal QMANs information. The array size is larger than
304  *                  the actual number of internal queues because they are not in
305  *                  consecutive order.
306  * @hbm_bar_cur_addr: current address of HBM PCI bar.
307  * @events: array that holds all event id's
308  * @events_stat: array that holds histogram of all received events.
309  * @events_stat_aggregate: same as events_stat but doesn't get cleared on reset
310  * @hw_cap_initialized: This field contains a bit per H/W engine. When that
311  *                      engine is initialized, that bit is set by the driver to
312  *                      signal we can use this engine in later code paths.
313  *                      Each bit is cleared upon reset of its corresponding H/W
314  *                      engine.
315  * @mmu_cache_inv_pi: PI for MMU cache invalidation flow. The H/W expects an
316  *                    8-bit value so use u8.
317  */
318 struct gaudi_device {
319 	int (*cpucp_info_get)(struct hl_device *hdev);
320 
321 	/* TODO: remove hw_queues_lock after moving to scheduler code */
322 	spinlock_t			hw_queues_lock;
323 
324 	struct gaudi_internal_qman_info	internal_qmans[GAUDI_QUEUE_ID_SIZE];
325 
326 	struct gaudi_collective_properties collective_props;
327 
328 	u64				hbm_bar_cur_addr;
329 
330 	u32				events[GAUDI_EVENT_SIZE];
331 	u32				events_stat[GAUDI_EVENT_SIZE];
332 	u32				events_stat_aggregate[GAUDI_EVENT_SIZE];
333 	u32				hw_cap_initialized;
334 	u8				mmu_cache_inv_pi;
335 };
336 
337 void gaudi_init_security(struct hl_device *hdev);
338 void gaudi_ack_protection_bits_errors(struct hl_device *hdev);
339 int gaudi_debug_coresight(struct hl_device *hdev, struct hl_ctx *ctx, void *data);
340 void gaudi_halt_coresight(struct hl_device *hdev, struct hl_ctx *ctx);
341 void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid);
342 
343 #endif /* GAUDIP_H_ */
344