1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 /* QLogic qed NIC Driver
3  * Copyright (c) 2015-2017  QLogic Corporation
4  * Copyright (c) 2019-2020 Marvell International Ltd.
5  */
6 
7 #include <linux/types.h>
8 #include <linux/io.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/errno.h>
12 #include <linux/kernel.h>
13 #include <linux/list.h>
14 #include <linux/mutex.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/string.h>
19 #include <linux/qed/qed_chain.h>
20 #include "qed.h"
21 #include "qed_hsi.h"
22 #include "qed_hw.h"
23 #include "qed_reg_addr.h"
24 #include "qed_sriov.h"
25 
26 #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT	1000
27 #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP		1000
28 #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT	100000
29 #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY		10
30 
31 /* Invalid values */
32 #define QED_BAR_INVALID_OFFSET          (cpu_to_le32(-1))
33 
34 struct qed_ptt {
35 	struct list_head	list_entry;
36 	unsigned int		idx;
37 	struct pxp_ptt_entry	pxp;
38 	u8			hwfn_id;
39 };
40 
41 struct qed_ptt_pool {
42 	struct list_head	free_list;
43 	spinlock_t		lock; /* ptt synchronized access */
44 	struct qed_ptt		ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM];
45 };
46 
47 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn)
48 {
49 	struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL);
50 	int i;
51 
52 	if (!p_pool)
53 		return -ENOMEM;
54 
55 	INIT_LIST_HEAD(&p_pool->free_list);
56 	for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
57 		p_pool->ptts[i].idx = i;
58 		p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET;
59 		p_pool->ptts[i].pxp.pretend.control = 0;
60 		p_pool->ptts[i].hwfn_id = p_hwfn->my_id;
61 		if (i >= RESERVED_PTT_MAX)
62 			list_add(&p_pool->ptts[i].list_entry,
63 				 &p_pool->free_list);
64 	}
65 
66 	p_hwfn->p_ptt_pool = p_pool;
67 	spin_lock_init(&p_pool->lock);
68 
69 	return 0;
70 }
71 
72 void qed_ptt_invalidate(struct qed_hwfn *p_hwfn)
73 {
74 	struct qed_ptt *p_ptt;
75 	int i;
76 
77 	for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
78 		p_ptt = &p_hwfn->p_ptt_pool->ptts[i];
79 		p_ptt->pxp.offset = QED_BAR_INVALID_OFFSET;
80 	}
81 }
82 
83 void qed_ptt_pool_free(struct qed_hwfn *p_hwfn)
84 {
85 	kfree(p_hwfn->p_ptt_pool);
86 	p_hwfn->p_ptt_pool = NULL;
87 }
88 
89 struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn)
90 {
91 	return qed_ptt_acquire_context(p_hwfn, false);
92 }
93 
94 struct qed_ptt *qed_ptt_acquire_context(struct qed_hwfn *p_hwfn, bool is_atomic)
95 {
96 	struct qed_ptt *p_ptt;
97 	unsigned int i, count;
98 
99 	if (is_atomic)
100 		count = QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT;
101 	else
102 		count = QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT;
103 
104 	/* Take the free PTT from the list */
105 	for (i = 0; i < count; i++) {
106 		spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
107 
108 		if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) {
109 			p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list,
110 						 struct qed_ptt, list_entry);
111 			list_del(&p_ptt->list_entry);
112 
113 			spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
114 
115 			DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
116 				   "allocated ptt %d\n", p_ptt->idx);
117 			return p_ptt;
118 		}
119 
120 		spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
121 
122 		if (is_atomic)
123 			udelay(QED_BAR_ACQUIRE_TIMEOUT_UDELAY);
124 		else
125 			usleep_range(QED_BAR_ACQUIRE_TIMEOUT_USLEEP,
126 				     QED_BAR_ACQUIRE_TIMEOUT_USLEEP * 2);
127 	}
128 
129 	DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n");
130 	return NULL;
131 }
132 
133 void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
134 {
135 	spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
136 	list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list);
137 	spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
138 }
139 
140 u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
141 {
142 	/* The HW is using DWORDS and we need to translate it to Bytes */
143 	return le32_to_cpu(p_ptt->pxp.offset) << 2;
144 }
145 
146 static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt)
147 {
148 	return PXP_PF_WINDOW_ADMIN_PER_PF_START +
149 	       p_ptt->idx * sizeof(struct pxp_ptt_entry);
150 }
151 
152 u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt)
153 {
154 	return PXP_EXTERNAL_BAR_PF_WINDOW_START +
155 	       p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE;
156 }
157 
158 void qed_ptt_set_win(struct qed_hwfn *p_hwfn,
159 		     struct qed_ptt *p_ptt, u32 new_hw_addr)
160 {
161 	u32 prev_hw_addr;
162 
163 	prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
164 
165 	if (new_hw_addr == prev_hw_addr)
166 		return;
167 
168 	/* Update PTT entery in admin window */
169 	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
170 		   "Updating PTT entry %d to offset 0x%x\n",
171 		   p_ptt->idx, new_hw_addr);
172 
173 	/* The HW is using DWORDS and the address is in Bytes */
174 	p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2);
175 
176 	REG_WR(p_hwfn,
177 	       qed_ptt_config_addr(p_ptt) +
178 	       offsetof(struct pxp_ptt_entry, offset),
179 	       le32_to_cpu(p_ptt->pxp.offset));
180 }
181 
182 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn,
183 		       struct qed_ptt *p_ptt, u32 hw_addr)
184 {
185 	u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
186 	u32 offset;
187 
188 	offset = hw_addr - win_hw_addr;
189 
190 	if (p_ptt->hwfn_id != p_hwfn->my_id)
191 		DP_NOTICE(p_hwfn,
192 			  "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n",
193 			  p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id);
194 
195 	/* Verify the address is within the window */
196 	if (hw_addr < win_hw_addr ||
197 	    offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) {
198 		qed_ptt_set_win(p_hwfn, p_ptt, hw_addr);
199 		offset = 0;
200 	}
201 
202 	return qed_ptt_get_bar_addr(p_ptt) + offset;
203 }
204 
205 struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn,
206 				     enum reserved_ptts ptt_idx)
207 {
208 	if (ptt_idx >= RESERVED_PTT_MAX) {
209 		DP_NOTICE(p_hwfn,
210 			  "Requested PTT %d is out of range\n", ptt_idx);
211 		return NULL;
212 	}
213 
214 	return &p_hwfn->p_ptt_pool->ptts[ptt_idx];
215 }
216 
217 void qed_wr(struct qed_hwfn *p_hwfn,
218 	    struct qed_ptt *p_ptt,
219 	    u32 hw_addr, u32 val)
220 {
221 	u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr);
222 
223 	REG_WR(p_hwfn, bar_addr, val);
224 	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
225 		   "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
226 		   bar_addr, hw_addr, val);
227 }
228 
229 u32 qed_rd(struct qed_hwfn *p_hwfn,
230 	   struct qed_ptt *p_ptt,
231 	   u32 hw_addr)
232 {
233 	u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr);
234 	u32 val = REG_RD(p_hwfn, bar_addr);
235 
236 	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
237 		   "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
238 		   bar_addr, hw_addr, val);
239 
240 	return val;
241 }
242 
243 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn,
244 			  struct qed_ptt *p_ptt,
245 			  void *addr, u32 hw_addr, size_t n, bool to_device)
246 {
247 	u32 dw_count, *host_addr, hw_offset;
248 	size_t quota, done = 0;
249 	u32 __iomem *reg_addr;
250 
251 	while (done < n) {
252 		quota = min_t(size_t, n - done,
253 			      PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE);
254 
255 		if (IS_PF(p_hwfn->cdev)) {
256 			qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done);
257 			hw_offset = qed_ptt_get_bar_addr(p_ptt);
258 		} else {
259 			hw_offset = hw_addr + done;
260 		}
261 
262 		dw_count = quota / 4;
263 		host_addr = (u32 *)((u8 *)addr + done);
264 		reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset);
265 		if (to_device)
266 			while (dw_count--)
267 				DIRECT_REG_WR(reg_addr++, *host_addr++);
268 		else
269 			while (dw_count--)
270 				*host_addr++ = DIRECT_REG_RD(reg_addr++);
271 
272 		done += quota;
273 	}
274 }
275 
276 void qed_memcpy_from(struct qed_hwfn *p_hwfn,
277 		     struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n)
278 {
279 	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
280 		   "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n",
281 		   hw_addr, dest, hw_addr, (unsigned long)n);
282 
283 	qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false);
284 }
285 
286 void qed_memcpy_to(struct qed_hwfn *p_hwfn,
287 		   struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n)
288 {
289 	DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
290 		   "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n",
291 		   hw_addr, hw_addr, src, (unsigned long)n);
292 
293 	qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true);
294 }
295 
296 void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid)
297 {
298 	u16 control = 0;
299 
300 	SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
301 	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
302 
303 	/* Every pretend undos previous pretends, including
304 	 * previous port pretend.
305 	 */
306 	SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
307 	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
308 	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
309 
310 	if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
311 		fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
312 
313 	p_ptt->pxp.pretend.control = cpu_to_le16(control);
314 	p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid);
315 
316 	REG_WR(p_hwfn,
317 	       qed_ptt_config_addr(p_ptt) +
318 	       offsetof(struct pxp_ptt_entry, pretend),
319 	       *(u32 *)&p_ptt->pxp.pretend);
320 }
321 
322 void qed_port_pretend(struct qed_hwfn *p_hwfn,
323 		      struct qed_ptt *p_ptt, u8 port_id)
324 {
325 	u16 control = 0;
326 
327 	SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
328 	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
329 	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
330 
331 	p_ptt->pxp.pretend.control = cpu_to_le16(control);
332 
333 	REG_WR(p_hwfn,
334 	       qed_ptt_config_addr(p_ptt) +
335 	       offsetof(struct pxp_ptt_entry, pretend),
336 	       *(u32 *)&p_ptt->pxp.pretend);
337 }
338 
339 void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
340 {
341 	u16 control = 0;
342 
343 	SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
344 	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
345 	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
346 
347 	p_ptt->pxp.pretend.control = cpu_to_le16(control);
348 
349 	REG_WR(p_hwfn,
350 	       qed_ptt_config_addr(p_ptt) +
351 	       offsetof(struct pxp_ptt_entry, pretend),
352 	       *(u32 *)&p_ptt->pxp.pretend);
353 }
354 
355 void qed_port_fid_pretend(struct qed_hwfn *p_hwfn,
356 			  struct qed_ptt *p_ptt, u8 port_id, u16 fid)
357 {
358 	u16 control = 0;
359 
360 	SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
361 	SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
362 	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
363 	SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
364 	SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
365 	if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
366 		fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
367 	p_ptt->pxp.pretend.control = cpu_to_le16(control);
368 	p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid);
369 	REG_WR(p_hwfn,
370 	       qed_ptt_config_addr(p_ptt) +
371 	       offsetof(struct pxp_ptt_entry, pretend),
372 	       *(u32 *)&p_ptt->pxp.pretend);
373 }
374 
375 u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid)
376 {
377 	u32 concrete_fid = 0;
378 
379 	SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id);
380 	SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid);
381 	SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1);
382 
383 	return concrete_fid;
384 }
385 
386 /* DMAE */
387 #define QED_DMAE_FLAGS_IS_SET(params, flag) \
388 	((params) != NULL && GET_FIELD((params)->flags, QED_DMAE_PARAMS_##flag))
389 
390 static void qed_dmae_opcode(struct qed_hwfn *p_hwfn,
391 			    const u8 is_src_type_grc,
392 			    const u8 is_dst_type_grc,
393 			    struct qed_dmae_params *p_params)
394 {
395 	u8 src_pfid, dst_pfid, port_id;
396 	u16 opcode_b = 0;
397 	u32 opcode = 0;
398 
399 	/* Whether the source is the PCIe or the GRC.
400 	 * 0- The source is the PCIe
401 	 * 1- The source is the GRC.
402 	 */
403 	SET_FIELD(opcode, DMAE_CMD_SRC,
404 		  (is_src_type_grc ? dmae_cmd_src_grc : dmae_cmd_src_pcie));
405 	src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, SRC_PF_VALID) ?
406 	    p_params->src_pfid : p_hwfn->rel_pf_id;
407 	SET_FIELD(opcode, DMAE_CMD_SRC_PF_ID, src_pfid);
408 
409 	/* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */
410 	SET_FIELD(opcode, DMAE_CMD_DST,
411 		  (is_dst_type_grc ? dmae_cmd_dst_grc : dmae_cmd_dst_pcie));
412 	dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, DST_PF_VALID) ?
413 	    p_params->dst_pfid : p_hwfn->rel_pf_id;
414 	SET_FIELD(opcode, DMAE_CMD_DST_PF_ID, dst_pfid);
415 
416 
417 	/* Whether to write a completion word to the completion destination:
418 	 * 0-Do not write a completion word
419 	 * 1-Write the completion word
420 	 */
421 	SET_FIELD(opcode, DMAE_CMD_COMP_WORD_EN, 1);
422 	SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1);
423 
424 	if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST))
425 		SET_FIELD(opcode, DMAE_CMD_COMP_FUNC, 1);
426 
427 	/* swapping mode 3 - big endian */
428 	SET_FIELD(opcode, DMAE_CMD_ENDIANITY_MODE, DMAE_CMD_ENDIANITY);
429 
430 	port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT_VALID)) ?
431 	    p_params->port_id : p_hwfn->port_id;
432 	SET_FIELD(opcode, DMAE_CMD_PORT_ID, port_id);
433 
434 	/* reset source address in next go */
435 	SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1);
436 
437 	/* reset dest address in next go */
438 	SET_FIELD(opcode, DMAE_CMD_DST_ADDR_RESET, 1);
439 
440 	/* SRC/DST VFID: all 1's - pf, otherwise VF id */
441 	if (QED_DMAE_FLAGS_IS_SET(p_params, SRC_VF_VALID)) {
442 		SET_FIELD(opcode, DMAE_CMD_SRC_VF_ID_VALID, 1);
443 		SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, p_params->src_vfid);
444 	} else {
445 		SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, 0xFF);
446 	}
447 	if (QED_DMAE_FLAGS_IS_SET(p_params, DST_VF_VALID)) {
448 		SET_FIELD(opcode, DMAE_CMD_DST_VF_ID_VALID, 1);
449 		SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, p_params->dst_vfid);
450 	} else {
451 		SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, 0xFF);
452 	}
453 
454 	p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode);
455 	p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b);
456 }
457 
458 u32 qed_dmae_idx_to_go_cmd(u8 idx)
459 {
460 	/* All the DMAE 'go' registers form an array in internal memory */
461 	return DMAE_REG_GO_C0 + (idx << 2);
462 }
463 
464 static int qed_dmae_post_command(struct qed_hwfn *p_hwfn,
465 				 struct qed_ptt *p_ptt)
466 {
467 	struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd;
468 	u8 idx_cmd = p_hwfn->dmae_info.channel, i;
469 	int qed_status = 0;
470 
471 	/* verify address is not NULL */
472 	if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) ||
473 	     ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) {
474 		DP_NOTICE(p_hwfn,
475 			  "source or destination address 0 idx_cmd=%d\n"
476 			  "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
477 			  idx_cmd,
478 			  le32_to_cpu(p_command->opcode),
479 			  le16_to_cpu(p_command->opcode_b),
480 			  le16_to_cpu(p_command->length_dw),
481 			  le32_to_cpu(p_command->src_addr_hi),
482 			  le32_to_cpu(p_command->src_addr_lo),
483 			  le32_to_cpu(p_command->dst_addr_hi),
484 			  le32_to_cpu(p_command->dst_addr_lo));
485 
486 		return -EINVAL;
487 	}
488 
489 	DP_VERBOSE(p_hwfn,
490 		   NETIF_MSG_HW,
491 		   "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
492 		   idx_cmd,
493 		   le32_to_cpu(p_command->opcode),
494 		   le16_to_cpu(p_command->opcode_b),
495 		   le16_to_cpu(p_command->length_dw),
496 		   le32_to_cpu(p_command->src_addr_hi),
497 		   le32_to_cpu(p_command->src_addr_lo),
498 		   le32_to_cpu(p_command->dst_addr_hi),
499 		   le32_to_cpu(p_command->dst_addr_lo));
500 
501 	/* Copy the command to DMAE - need to do it before every call
502 	 * for source/dest address no reset.
503 	 * The first 9 DWs are the command registers, the 10 DW is the
504 	 * GO register, and the rest are result registers
505 	 * (which are read only by the client).
506 	 */
507 	for (i = 0; i < DMAE_CMD_SIZE; i++) {
508 		u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ?
509 			   *(((u32 *)p_command) + i) : 0;
510 
511 		qed_wr(p_hwfn, p_ptt,
512 		       DMAE_REG_CMD_MEM +
513 		       (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) +
514 		       (i * sizeof(u32)), data);
515 	}
516 
517 	qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE);
518 
519 	return qed_status;
520 }
521 
522 int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn)
523 {
524 	dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr;
525 	struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd;
526 	u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer;
527 	u32 **p_comp = &p_hwfn->dmae_info.p_completion_word;
528 
529 	*p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
530 				     sizeof(u32), p_addr, GFP_KERNEL);
531 	if (!*p_comp)
532 		goto err;
533 
534 	p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr;
535 	*p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
536 				    sizeof(struct dmae_cmd),
537 				    p_addr, GFP_KERNEL);
538 	if (!*p_cmd)
539 		goto err;
540 
541 	p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr;
542 	*p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
543 				     sizeof(u32) * DMAE_MAX_RW_SIZE,
544 				     p_addr, GFP_KERNEL);
545 	if (!*p_buff)
546 		goto err;
547 
548 	p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id;
549 
550 	return 0;
551 err:
552 	qed_dmae_info_free(p_hwfn);
553 	return -ENOMEM;
554 }
555 
556 void qed_dmae_info_free(struct qed_hwfn *p_hwfn)
557 {
558 	dma_addr_t p_phys;
559 
560 	/* Just make sure no one is in the middle */
561 	mutex_lock(&p_hwfn->dmae_info.mutex);
562 
563 	if (p_hwfn->dmae_info.p_completion_word) {
564 		p_phys = p_hwfn->dmae_info.completion_word_phys_addr;
565 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
566 				  sizeof(u32),
567 				  p_hwfn->dmae_info.p_completion_word, p_phys);
568 		p_hwfn->dmae_info.p_completion_word = NULL;
569 	}
570 
571 	if (p_hwfn->dmae_info.p_dmae_cmd) {
572 		p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr;
573 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
574 				  sizeof(struct dmae_cmd),
575 				  p_hwfn->dmae_info.p_dmae_cmd, p_phys);
576 		p_hwfn->dmae_info.p_dmae_cmd = NULL;
577 	}
578 
579 	if (p_hwfn->dmae_info.p_intermediate_buffer) {
580 		p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
581 		dma_free_coherent(&p_hwfn->cdev->pdev->dev,
582 				  sizeof(u32) * DMAE_MAX_RW_SIZE,
583 				  p_hwfn->dmae_info.p_intermediate_buffer,
584 				  p_phys);
585 		p_hwfn->dmae_info.p_intermediate_buffer = NULL;
586 	}
587 
588 	mutex_unlock(&p_hwfn->dmae_info.mutex);
589 }
590 
591 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn)
592 {
593 	u32 wait_cnt_limit = 10000, wait_cnt = 0;
594 	int qed_status = 0;
595 
596 	barrier();
597 	while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) {
598 		udelay(DMAE_MIN_WAIT_TIME);
599 		if (++wait_cnt > wait_cnt_limit) {
600 			DP_NOTICE(p_hwfn->cdev,
601 				  "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n",
602 				  *p_hwfn->dmae_info.p_completion_word,
603 				 DMAE_COMPLETION_VAL);
604 			qed_status = -EBUSY;
605 			break;
606 		}
607 
608 		/* to sync the completion_word since we are not
609 		 * using the volatile keyword for p_completion_word
610 		 */
611 		barrier();
612 	}
613 
614 	if (qed_status == 0)
615 		*p_hwfn->dmae_info.p_completion_word = 0;
616 
617 	return qed_status;
618 }
619 
620 static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
621 					  struct qed_ptt *p_ptt,
622 					  u64 src_addr,
623 					  u64 dst_addr,
624 					  u8 src_type,
625 					  u8 dst_type,
626 					  u32 length_dw)
627 {
628 	dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
629 	struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
630 	int qed_status = 0;
631 
632 	switch (src_type) {
633 	case QED_DMAE_ADDRESS_GRC:
634 	case QED_DMAE_ADDRESS_HOST_PHYS:
635 		cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr));
636 		cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr));
637 		break;
638 	/* for virtual source addresses we use the intermediate buffer. */
639 	case QED_DMAE_ADDRESS_HOST_VIRT:
640 		cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys));
641 		cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys));
642 		memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0],
643 		       (void *)(uintptr_t)src_addr,
644 		       length_dw * sizeof(u32));
645 		break;
646 	default:
647 		return -EINVAL;
648 	}
649 
650 	switch (dst_type) {
651 	case QED_DMAE_ADDRESS_GRC:
652 	case QED_DMAE_ADDRESS_HOST_PHYS:
653 		cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr));
654 		cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr));
655 		break;
656 	/* for virtual source addresses we use the intermediate buffer. */
657 	case QED_DMAE_ADDRESS_HOST_VIRT:
658 		cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys));
659 		cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys));
660 		break;
661 	default:
662 		return -EINVAL;
663 	}
664 
665 	cmd->length_dw = cpu_to_le16((u16)length_dw);
666 
667 	qed_dmae_post_command(p_hwfn, p_ptt);
668 
669 	qed_status = qed_dmae_operation_wait(p_hwfn);
670 
671 	if (qed_status) {
672 		DP_NOTICE(p_hwfn,
673 			  "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n",
674 			  src_addr, dst_addr, length_dw);
675 		return qed_status;
676 	}
677 
678 	if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT)
679 		memcpy((void *)(uintptr_t)(dst_addr),
680 		       &p_hwfn->dmae_info.p_intermediate_buffer[0],
681 		       length_dw * sizeof(u32));
682 
683 	return 0;
684 }
685 
686 static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn,
687 				    struct qed_ptt *p_ptt,
688 				    u64 src_addr, u64 dst_addr,
689 				    u8 src_type, u8 dst_type,
690 				    u32 size_in_dwords,
691 				    struct qed_dmae_params *p_params)
692 {
693 	dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr;
694 	u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0;
695 	struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
696 	u64 src_addr_split = 0, dst_addr_split = 0;
697 	u16 length_limit = DMAE_MAX_RW_SIZE;
698 	int qed_status = 0;
699 	u32 offset = 0;
700 
701 	if (p_hwfn->cdev->recov_in_prog) {
702 		DP_VERBOSE(p_hwfn,
703 			   NETIF_MSG_HW,
704 			   "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n",
705 			   src_addr, src_type, dst_addr, dst_type,
706 			   size_in_dwords);
707 
708 		/* Let the flow complete w/o any error handling */
709 		return 0;
710 	}
711 
712 	qed_dmae_opcode(p_hwfn,
713 			(src_type == QED_DMAE_ADDRESS_GRC),
714 			(dst_type == QED_DMAE_ADDRESS_GRC),
715 			p_params);
716 
717 	cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys));
718 	cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys));
719 	cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL);
720 
721 	/* Check if the grc_addr is valid like < MAX_GRC_OFFSET */
722 	cnt_split = size_in_dwords / length_limit;
723 	length_mod = size_in_dwords % length_limit;
724 
725 	src_addr_split = src_addr;
726 	dst_addr_split = dst_addr;
727 
728 	for (i = 0; i <= cnt_split; i++) {
729 		offset = length_limit * i;
730 
731 		if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) {
732 			if (src_type == QED_DMAE_ADDRESS_GRC)
733 				src_addr_split = src_addr + offset;
734 			else
735 				src_addr_split = src_addr + (offset * 4);
736 		}
737 
738 		if (dst_type == QED_DMAE_ADDRESS_GRC)
739 			dst_addr_split = dst_addr + offset;
740 		else
741 			dst_addr_split = dst_addr + (offset * 4);
742 
743 		length_cur = (cnt_split == i) ? length_mod : length_limit;
744 
745 		/* might be zero on last iteration */
746 		if (!length_cur)
747 			continue;
748 
749 		qed_status = qed_dmae_execute_sub_operation(p_hwfn,
750 							    p_ptt,
751 							    src_addr_split,
752 							    dst_addr_split,
753 							    src_type,
754 							    dst_type,
755 							    length_cur);
756 		if (qed_status) {
757 			qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_DMAE_FAIL,
758 					  "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n",
759 					  qed_status, src_addr,
760 					  dst_addr, length_cur);
761 			break;
762 		}
763 	}
764 
765 	return qed_status;
766 }
767 
768 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn,
769 		      struct qed_ptt *p_ptt,
770 		      u64 source_addr, u32 grc_addr, u32 size_in_dwords,
771 		      struct qed_dmae_params *p_params)
772 {
773 	u32 grc_addr_in_dw = grc_addr / sizeof(u32);
774 	int rc;
775 
776 
777 	mutex_lock(&p_hwfn->dmae_info.mutex);
778 
779 	rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr,
780 				      grc_addr_in_dw,
781 				      QED_DMAE_ADDRESS_HOST_VIRT,
782 				      QED_DMAE_ADDRESS_GRC,
783 				      size_in_dwords, p_params);
784 
785 	mutex_unlock(&p_hwfn->dmae_info.mutex);
786 
787 	return rc;
788 }
789 
790 int qed_dmae_grc2host(struct qed_hwfn *p_hwfn,
791 		      struct qed_ptt *p_ptt,
792 		      u32 grc_addr,
793 		      dma_addr_t dest_addr, u32 size_in_dwords,
794 		      struct qed_dmae_params *p_params)
795 {
796 	u32 grc_addr_in_dw = grc_addr / sizeof(u32);
797 	int rc;
798 
799 
800 	mutex_lock(&p_hwfn->dmae_info.mutex);
801 
802 	rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw,
803 				      dest_addr, QED_DMAE_ADDRESS_GRC,
804 				      QED_DMAE_ADDRESS_HOST_VIRT,
805 				      size_in_dwords, p_params);
806 
807 	mutex_unlock(&p_hwfn->dmae_info.mutex);
808 
809 	return rc;
810 }
811 
812 int qed_dmae_host2host(struct qed_hwfn *p_hwfn,
813 		       struct qed_ptt *p_ptt,
814 		       dma_addr_t source_addr,
815 		       dma_addr_t dest_addr,
816 		       u32 size_in_dwords, struct qed_dmae_params *p_params)
817 {
818 	int rc;
819 
820 	mutex_lock(&(p_hwfn->dmae_info.mutex));
821 
822 	rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr,
823 				      dest_addr,
824 				      QED_DMAE_ADDRESS_HOST_PHYS,
825 				      QED_DMAE_ADDRESS_HOST_PHYS,
826 				      size_in_dwords, p_params);
827 
828 	mutex_unlock(&(p_hwfn->dmae_info.mutex));
829 
830 	return rc;
831 }
832 
833 void qed_hw_err_notify(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
834 		       enum qed_hw_err_type err_type, const char *fmt, ...)
835 {
836 	char buf[QED_HW_ERR_MAX_STR_SIZE];
837 	va_list vl;
838 	int len;
839 
840 	if (fmt) {
841 		va_start(vl, fmt);
842 		len = vsnprintf(buf, QED_HW_ERR_MAX_STR_SIZE, fmt, vl);
843 		va_end(vl);
844 
845 		if (len > QED_HW_ERR_MAX_STR_SIZE - 1)
846 			len = QED_HW_ERR_MAX_STR_SIZE - 1;
847 
848 		DP_NOTICE(p_hwfn, "%s", buf);
849 	}
850 
851 	/* Fan failure cannot be masked by handling of another HW error */
852 	if (p_hwfn->cdev->recov_in_prog &&
853 	    err_type != QED_HW_ERR_FAN_FAIL) {
854 		DP_VERBOSE(p_hwfn,
855 			   NETIF_MSG_DRV,
856 			   "Recovery is in progress. Avoid notifying about HW error %d.\n",
857 			   err_type);
858 		return;
859 	}
860 
861 	qed_hw_error_occurred(p_hwfn, err_type);
862 
863 	if (fmt)
864 		qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, buf, len);
865 }
866 
867 int qed_dmae_sanity(struct qed_hwfn *p_hwfn,
868 		    struct qed_ptt *p_ptt, const char *phase)
869 {
870 	u32 size = PAGE_SIZE / 2, val;
871 	int rc = 0;
872 	dma_addr_t p_phys;
873 	void *p_virt;
874 	u32 *p_tmp;
875 
876 	p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
877 				    2 * size, &p_phys, GFP_KERNEL);
878 	if (!p_virt) {
879 		DP_NOTICE(p_hwfn,
880 			  "DMAE sanity [%s]: failed to allocate memory\n",
881 			  phase);
882 		return -ENOMEM;
883 	}
884 
885 	/* Fill the bottom half of the allocated memory with a known pattern */
886 	for (p_tmp = (u32 *)p_virt;
887 	     p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) {
888 		/* Save the address itself as the value */
889 		val = (u32)(uintptr_t)p_tmp;
890 		*p_tmp = val;
891 	}
892 
893 	/* Zero the top half of the allocated memory */
894 	memset((u8 *)p_virt + size, 0, size);
895 
896 	DP_VERBOSE(p_hwfn,
897 		   QED_MSG_SP,
898 		   "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n",
899 		   phase,
900 		   (u64)p_phys,
901 		   p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size);
902 
903 	rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size,
904 				size / 4, NULL);
905 	if (rc) {
906 		DP_NOTICE(p_hwfn,
907 			  "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n",
908 			  phase, rc);
909 		goto out;
910 	}
911 
912 	/* Verify that the top half of the allocated memory has the pattern */
913 	for (p_tmp = (u32 *)((u8 *)p_virt + size);
914 	     p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) {
915 		/* The corresponding address in the bottom half */
916 		val = (u32)(uintptr_t)p_tmp - size;
917 
918 		if (*p_tmp != val) {
919 			DP_NOTICE(p_hwfn,
920 				  "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n",
921 				  phase,
922 				  (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt),
923 				  p_tmp, *p_tmp, val);
924 			rc = -EINVAL;
925 			goto out;
926 		}
927 	}
928 
929 out:
930 	dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys);
931 	return rc;
932 }
933