1 /*
2  * Huawei HiNIC PCI Express Linux driver
3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12  * for more details.
13  *
14  */
15 
16 #include <linux/kernel.h>
17 #include <linux/types.h>
18 #include <linux/errno.h>
19 #include <linux/pci.h>
20 #include <linux/device.h>
21 #include <linux/slab.h>
22 #include <linux/vmalloc.h>
23 #include <linux/spinlock.h>
24 #include <linux/sizes.h>
25 #include <linux/atomic.h>
26 #include <linux/log2.h>
27 #include <linux/io.h>
28 #include <linux/completion.h>
29 #include <linux/err.h>
30 #include <asm/byteorder.h>
31 #include <asm/barrier.h>
32 
33 #include "hinic_common.h"
34 #include "hinic_hw_if.h"
35 #include "hinic_hw_eqs.h"
36 #include "hinic_hw_mgmt.h"
37 #include "hinic_hw_wqe.h"
38 #include "hinic_hw_wq.h"
39 #include "hinic_hw_cmdq.h"
40 #include "hinic_hw_io.h"
41 #include "hinic_hw_dev.h"
42 
43 #define CMDQ_CEQE_TYPE_SHIFT                    0
44 
45 #define CMDQ_CEQE_TYPE_MASK                     0x7
46 
47 #define CMDQ_CEQE_GET(val, member)              \
48 			(((val) >> CMDQ_CEQE_##member##_SHIFT) \
49 			 & CMDQ_CEQE_##member##_MASK)
50 
51 #define CMDQ_WQE_ERRCODE_VAL_SHIFT              20
52 
53 #define CMDQ_WQE_ERRCODE_VAL_MASK               0xF
54 
55 #define CMDQ_WQE_ERRCODE_GET(val, member)       \
56 			(((val) >> CMDQ_WQE_ERRCODE_##member##_SHIFT) \
57 			 & CMDQ_WQE_ERRCODE_##member##_MASK)
58 
59 #define CMDQ_DB_PI_OFF(pi)              (((u16)LOWER_8_BITS(pi)) << 3)
60 
61 #define CMDQ_DB_ADDR(db_base, pi)       ((db_base) + CMDQ_DB_PI_OFF(pi))
62 
63 #define CMDQ_WQE_HEADER(wqe)            ((struct hinic_cmdq_header *)(wqe))
64 
65 #define CMDQ_WQE_COMPLETED(ctrl_info)   \
66 			HINIC_CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT)
67 
68 #define FIRST_DATA_TO_WRITE_LAST        sizeof(u64)
69 
70 #define CMDQ_DB_OFF                     SZ_2K
71 
72 #define CMDQ_WQEBB_SIZE                 64
73 #define CMDQ_WQE_SIZE                   64
74 #define CMDQ_DEPTH                      SZ_4K
75 
76 #define CMDQ_WQ_PAGE_SIZE               SZ_4K
77 
78 #define WQE_LCMD_SIZE                   64
79 #define WQE_SCMD_SIZE                   64
80 
81 #define COMPLETE_LEN                    3
82 
83 #define CMDQ_TIMEOUT                    1000
84 
85 #define CMDQ_PFN(addr, page_size)       ((addr) >> (ilog2(page_size)))
86 
87 #define cmdq_to_cmdqs(cmdq)     container_of((cmdq) - (cmdq)->cmdq_type, \
88 					     struct hinic_cmdqs, cmdq[0])
89 
90 #define cmdqs_to_func_to_io(cmdqs)      container_of(cmdqs, \
91 						     struct hinic_func_to_io, \
92 						     cmdqs)
93 
94 enum cmdq_wqe_type {
95 	WQE_LCMD_TYPE = 0,
96 	WQE_SCMD_TYPE = 1,
97 };
98 
99 enum completion_format {
100 	COMPLETE_DIRECT = 0,
101 	COMPLETE_SGE    = 1,
102 };
103 
104 enum data_format {
105 	DATA_SGE        = 0,
106 	DATA_DIRECT     = 1,
107 };
108 
109 enum bufdesc_len {
110 	BUFDESC_LCMD_LEN = 2,   /* 16 bytes - 2(8 byte unit) */
111 	BUFDESC_SCMD_LEN = 3,   /* 24 bytes - 3(8 byte unit) */
112 };
113 
114 enum ctrl_sect_len {
115 	CTRL_SECT_LEN        = 1, /* 4 bytes (ctrl) - 1(8 byte unit) */
116 	CTRL_DIRECT_SECT_LEN = 2, /* 12 bytes (ctrl + rsvd) - 2(8 byte unit) */
117 };
118 
119 enum cmdq_scmd_type {
120 	CMDQ_SET_ARM_CMD = 2,
121 };
122 
123 enum cmdq_cmd_type {
124 	CMDQ_CMD_SYNC_DIRECT_RESP = 0,
125 	CMDQ_CMD_SYNC_SGE_RESP    = 1,
126 };
127 
128 enum completion_request {
129 	NO_CEQ  = 0,
130 	CEQ_SET = 1,
131 };
132 
133 /**
134  * hinic_alloc_cmdq_buf - alloc buffer for sending command
135  * @cmdqs: the cmdqs
136  * @cmdq_buf: the buffer returned in this struct
137  *
138  * Return 0 - Success, negative - Failure
139  **/
140 int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs,
141 			 struct hinic_cmdq_buf *cmdq_buf)
142 {
143 	struct hinic_hwif *hwif = cmdqs->hwif;
144 	struct pci_dev *pdev = hwif->pdev;
145 
146 	cmdq_buf->buf = dma_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL,
147 				       &cmdq_buf->dma_addr);
148 	if (!cmdq_buf->buf) {
149 		dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n");
150 		return -ENOMEM;
151 	}
152 
153 	return 0;
154 }
155 
156 /**
157  * hinic_free_cmdq_buf - free buffer
158  * @cmdqs: the cmdqs
159  * @cmdq_buf: the buffer to free that is in this struct
160  **/
161 void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs,
162 			 struct hinic_cmdq_buf *cmdq_buf)
163 {
164 	dma_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr);
165 }
166 
167 static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len)
168 {
169 	unsigned int wqe_size = 0;
170 
171 	switch (len) {
172 	case BUFDESC_LCMD_LEN:
173 		wqe_size = WQE_LCMD_SIZE;
174 		break;
175 	case BUFDESC_SCMD_LEN:
176 		wqe_size = WQE_SCMD_SIZE;
177 		break;
178 	}
179 
180 	return wqe_size;
181 }
182 
183 static void cmdq_set_sge_completion(struct hinic_cmdq_completion *completion,
184 				    struct hinic_cmdq_buf *buf_out)
185 {
186 	struct hinic_sge_resp *sge_resp = &completion->sge_resp;
187 
188 	hinic_set_sge(&sge_resp->sge, buf_out->dma_addr, buf_out->size);
189 }
190 
191 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped,
192 				  enum hinic_cmd_ack_type ack_type,
193 				  enum hinic_mod_type mod, u8 cmd, u16 prod_idx,
194 				  enum completion_format complete_format,
195 				  enum data_format data_format,
196 				  enum bufdesc_len buf_len)
197 {
198 	struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
199 	struct hinic_cmdq_wqe_scmd *wqe_scmd;
200 	enum ctrl_sect_len ctrl_len;
201 	struct hinic_ctrl *ctrl;
202 	u32 saved_data;
203 
204 	if (data_format == DATA_SGE) {
205 		wqe_lcmd = &wqe->wqe_lcmd;
206 
207 		wqe_lcmd->status.status_info = 0;
208 		ctrl = &wqe_lcmd->ctrl;
209 		ctrl_len = CTRL_SECT_LEN;
210 	} else {
211 		wqe_scmd = &wqe->direct_wqe.wqe_scmd;
212 
213 		wqe_scmd->status.status_info = 0;
214 		ctrl = &wqe_scmd->ctrl;
215 		ctrl_len = CTRL_DIRECT_SECT_LEN;
216 	}
217 
218 	ctrl->ctrl_info = HINIC_CMDQ_CTRL_SET(prod_idx, PI)             |
219 			  HINIC_CMDQ_CTRL_SET(cmd, CMD)                 |
220 			  HINIC_CMDQ_CTRL_SET(mod, MOD)                 |
221 			  HINIC_CMDQ_CTRL_SET(ack_type, ACK_TYPE);
222 
223 	CMDQ_WQE_HEADER(wqe)->header_info =
224 		HINIC_CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN)            |
225 		HINIC_CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT)   |
226 		HINIC_CMDQ_WQE_HEADER_SET(data_format, DATA_FMT)           |
227 		HINIC_CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ)           |
228 		HINIC_CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) |
229 		HINIC_CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN)              |
230 		HINIC_CMDQ_WQE_HEADER_SET(wrapped, TOGGLED_WRAPPED);
231 
232 	saved_data = CMDQ_WQE_HEADER(wqe)->saved_data;
233 	saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM);
234 
235 	if ((cmd == CMDQ_SET_ARM_CMD) && (mod == HINIC_MOD_COMM))
236 		CMDQ_WQE_HEADER(wqe)->saved_data |=
237 						HINIC_SAVED_DATA_SET(1, ARM);
238 	else
239 		CMDQ_WQE_HEADER(wqe)->saved_data = saved_data;
240 }
241 
242 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe_lcmd,
243 				  struct hinic_cmdq_buf *buf_in)
244 {
245 	hinic_set_sge(&wqe_lcmd->buf_desc.sge, buf_in->dma_addr, buf_in->size);
246 }
247 
248 static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe,
249 				     void *buf_in, u32 in_size)
250 {
251 	struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd;
252 
253 	wqe_scmd->buf_desc.buf_len = in_size;
254 	memcpy(wqe_scmd->buf_desc.data, buf_in, in_size);
255 }
256 
257 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe,
258 			      enum cmdq_cmd_type cmd_type,
259 			      struct hinic_cmdq_buf *buf_in,
260 			      struct hinic_cmdq_buf *buf_out, int wrapped,
261 			      enum hinic_cmd_ack_type ack_type,
262 			      enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
263 {
264 	struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
265 	enum completion_format complete_format;
266 
267 	switch (cmd_type) {
268 	case CMDQ_CMD_SYNC_SGE_RESP:
269 		complete_format = COMPLETE_SGE;
270 		cmdq_set_sge_completion(&wqe_lcmd->completion, buf_out);
271 		break;
272 	case CMDQ_CMD_SYNC_DIRECT_RESP:
273 		complete_format = COMPLETE_DIRECT;
274 		wqe_lcmd->completion.direct_resp = 0;
275 		break;
276 	}
277 
278 	cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd,
279 			      prod_idx, complete_format, DATA_SGE,
280 			      BUFDESC_LCMD_LEN);
281 
282 	cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in);
283 }
284 
285 static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe,
286 				enum cmdq_cmd_type cmd_type,
287 				void *buf_in, u16 in_size,
288 				struct hinic_cmdq_buf *buf_out, int wrapped,
289 				enum hinic_cmd_ack_type ack_type,
290 				enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
291 {
292 	struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
293 	enum completion_format complete_format;
294 	struct hinic_cmdq_wqe_scmd *wqe_scmd;
295 
296 	wqe_scmd = &direct_wqe->wqe_scmd;
297 
298 	switch (cmd_type) {
299 	case CMDQ_CMD_SYNC_SGE_RESP:
300 		complete_format = COMPLETE_SGE;
301 		cmdq_set_sge_completion(&wqe_scmd->completion, buf_out);
302 		break;
303 	case CMDQ_CMD_SYNC_DIRECT_RESP:
304 		complete_format = COMPLETE_DIRECT;
305 		wqe_scmd->completion.direct_resp = 0;
306 		break;
307 	}
308 
309 	cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx,
310 			      complete_format, DATA_DIRECT, BUFDESC_SCMD_LEN);
311 
312 	cmdq_set_direct_wqe_data(direct_wqe, buf_in, in_size);
313 }
314 
315 static void cmdq_wqe_fill(void *dst, void *src)
316 {
317 	memcpy(dst + FIRST_DATA_TO_WRITE_LAST, src + FIRST_DATA_TO_WRITE_LAST,
318 	       CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST);
319 
320 	wmb();          /* The first 8 bytes should be written last */
321 
322 	*(u64 *)dst = *(u64 *)src;
323 }
324 
325 static void cmdq_fill_db(u32 *db_info,
326 			 enum hinic_cmdq_type cmdq_type, u16 prod_idx)
327 {
328 	*db_info = HINIC_CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) |
329 		   HINIC_CMDQ_DB_INFO_SET(HINIC_CTRL_PATH, PATH)               |
330 		   HINIC_CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE)                |
331 		   HINIC_CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, DB_TYPE);
332 }
333 
334 static void cmdq_set_db(struct hinic_cmdq *cmdq,
335 			enum hinic_cmdq_type cmdq_type, u16 prod_idx)
336 {
337 	u32 db_info;
338 
339 	cmdq_fill_db(&db_info, cmdq_type, prod_idx);
340 
341 	/* The data that is written to HW should be in Big Endian Format */
342 	db_info = cpu_to_be32(db_info);
343 
344 	wmb();  /* write all before the doorbell */
345 
346 	writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx));
347 }
348 
349 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,
350 				     enum hinic_mod_type mod, u8 cmd,
351 				     struct hinic_cmdq_buf *buf_in,
352 				     u64 *resp)
353 {
354 	struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe;
355 	u16 curr_prod_idx, next_prod_idx;
356 	int errcode, wrapped, num_wqebbs;
357 	struct hinic_wq *wq = cmdq->wq;
358 	struct hinic_hw_wqe *hw_wqe;
359 	struct completion done;
360 
361 	/* Keep doorbell index correct. bh - for tasklet(ceq). */
362 	spin_lock_bh(&cmdq->cmdq_lock);
363 
364 	/* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/
365 	hw_wqe = hinic_get_wqe(wq, WQE_LCMD_SIZE, &curr_prod_idx);
366 	if (IS_ERR(hw_wqe)) {
367 		spin_unlock_bh(&cmdq->cmdq_lock);
368 		return -EBUSY;
369 	}
370 
371 	curr_cmdq_wqe = &hw_wqe->cmdq_wqe;
372 
373 	wrapped = cmdq->wrapped;
374 
375 	num_wqebbs = ALIGN(WQE_LCMD_SIZE, wq->wqebb_size) / wq->wqebb_size;
376 	next_prod_idx = curr_prod_idx + num_wqebbs;
377 	if (next_prod_idx >= wq->q_depth) {
378 		cmdq->wrapped = !cmdq->wrapped;
379 		next_prod_idx -= wq->q_depth;
380 	}
381 
382 	cmdq->errcode[curr_prod_idx] = &errcode;
383 
384 	init_completion(&done);
385 	cmdq->done[curr_prod_idx] = &done;
386 
387 	cmdq_set_lcmd_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, NULL,
388 			  wrapped, HINIC_CMD_ACK_TYPE_CMDQ, mod, cmd,
389 			  curr_prod_idx);
390 
391 	/* The data that is written to HW should be in Big Endian Format */
392 	hinic_cpu_to_be32(&cmdq_wqe, WQE_LCMD_SIZE);
393 
394 	/* CMDQ WQE is not shadow, therefore wqe will be written to wq */
395 	cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe);
396 
397 	cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
398 
399 	spin_unlock_bh(&cmdq->cmdq_lock);
400 
401 	if (!wait_for_completion_timeout(&done, CMDQ_TIMEOUT)) {
402 		spin_lock_bh(&cmdq->cmdq_lock);
403 
404 		if (cmdq->errcode[curr_prod_idx] == &errcode)
405 			cmdq->errcode[curr_prod_idx] = NULL;
406 
407 		if (cmdq->done[curr_prod_idx] == &done)
408 			cmdq->done[curr_prod_idx] = NULL;
409 
410 		spin_unlock_bh(&cmdq->cmdq_lock);
411 
412 		return -ETIMEDOUT;
413 	}
414 
415 	smp_rmb();      /* read error code after completion */
416 
417 	if (resp) {
418 		struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &curr_cmdq_wqe->wqe_lcmd;
419 
420 		*resp = cpu_to_be64(wqe_lcmd->completion.direct_resp);
421 	}
422 
423 	if (errcode != 0)
424 		return -EFAULT;
425 
426 	return 0;
427 }
428 
429 static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in,
430 			    u16 in_size)
431 {
432 	struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe;
433 	u16 curr_prod_idx, next_prod_idx;
434 	struct hinic_wq *wq = cmdq->wq;
435 	struct hinic_hw_wqe *hw_wqe;
436 	int wrapped, num_wqebbs;
437 
438 	/* Keep doorbell index correct */
439 	spin_lock(&cmdq->cmdq_lock);
440 
441 	/* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/
442 	hw_wqe = hinic_get_wqe(wq, WQE_SCMD_SIZE, &curr_prod_idx);
443 	if (IS_ERR(hw_wqe)) {
444 		spin_unlock(&cmdq->cmdq_lock);
445 		return -EBUSY;
446 	}
447 
448 	curr_cmdq_wqe = &hw_wqe->cmdq_wqe;
449 
450 	wrapped = cmdq->wrapped;
451 
452 	num_wqebbs = ALIGN(WQE_SCMD_SIZE, wq->wqebb_size) / wq->wqebb_size;
453 	next_prod_idx = curr_prod_idx + num_wqebbs;
454 	if (next_prod_idx >= wq->q_depth) {
455 		cmdq->wrapped = !cmdq->wrapped;
456 		next_prod_idx -= wq->q_depth;
457 	}
458 
459 	cmdq_set_direct_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in,
460 			    in_size, NULL, wrapped, HINIC_CMD_ACK_TYPE_CMDQ,
461 			    HINIC_MOD_COMM, CMDQ_SET_ARM_CMD, curr_prod_idx);
462 
463 	/* The data that is written to HW should be in Big Endian Format */
464 	hinic_cpu_to_be32(&cmdq_wqe, WQE_SCMD_SIZE);
465 
466 	/* cmdq wqe is not shadow, therefore wqe will be written to wq */
467 	cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe);
468 
469 	cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
470 
471 	spin_unlock(&cmdq->cmdq_lock);
472 	return 0;
473 }
474 
475 static int cmdq_params_valid(struct hinic_cmdq_buf *buf_in)
476 {
477 	if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE)
478 		return -EINVAL;
479 
480 	return 0;
481 }
482 
483 /**
484  * hinic_cmdq_direct_resp - send command with direct data as resp
485  * @cmdqs: the cmdqs
486  * @mod: module on the card that will handle the command
487  * @cmd: the command
488  * @buf_in: the buffer for the command
489  * @resp: the response to return
490  *
491  * Return 0 - Success, negative - Failure
492  **/
493 int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs,
494 			   enum hinic_mod_type mod, u8 cmd,
495 			   struct hinic_cmdq_buf *buf_in, u64 *resp)
496 {
497 	struct hinic_hwif *hwif = cmdqs->hwif;
498 	struct pci_dev *pdev = hwif->pdev;
499 	int err;
500 
501 	err = cmdq_params_valid(buf_in);
502 	if (err) {
503 		dev_err(&pdev->dev, "Invalid CMDQ parameters\n");
504 		return err;
505 	}
506 
507 	return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC],
508 					 mod, cmd, buf_in, resp);
509 }
510 
511 /**
512  * hinic_set_arm_bit - set arm bit for enable interrupt again
513  * @cmdqs: the cmdqs
514  * @q_type: type of queue to set the arm bit for
515  * @q_id: the queue number
516  *
517  * Return 0 - Success, negative - Failure
518  **/
519 int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs,
520 		      enum hinic_set_arm_qtype q_type, u32 q_id)
521 {
522 	struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC];
523 	struct hinic_hwif *hwif = cmdqs->hwif;
524 	struct pci_dev *pdev = hwif->pdev;
525 	struct hinic_cmdq_arm_bit arm_bit;
526 	int err;
527 
528 	arm_bit.q_type = q_type;
529 	arm_bit.q_id   = q_id;
530 
531 	err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit));
532 	if (err) {
533 		dev_err(&pdev->dev, "Failed to set arm for qid %d\n", q_id);
534 		return err;
535 	}
536 
537 	return 0;
538 }
539 
540 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq,
541 				   struct hinic_cmdq_wqe *wqe)
542 {
543 	u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info);
544 	unsigned int bufdesc_len, wqe_size;
545 	struct hinic_ctrl *ctrl;
546 
547 	bufdesc_len = HINIC_CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN);
548 	wqe_size = cmdq_wqe_size_from_bdlen(bufdesc_len);
549 	if (wqe_size == WQE_LCMD_SIZE) {
550 		struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
551 
552 		ctrl = &wqe_lcmd->ctrl;
553 	} else {
554 		struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
555 		struct hinic_cmdq_wqe_scmd *wqe_scmd;
556 
557 		wqe_scmd = &direct_wqe->wqe_scmd;
558 		ctrl = &wqe_scmd->ctrl;
559 	}
560 
561 	/* clear HW busy bit */
562 	ctrl->ctrl_info = 0;
563 
564 	wmb();  /* verify wqe is clear */
565 }
566 
567 /**
568  * cmdq_arm_ceq_handler - cmdq completion event handler for arm command
569  * @cmdq: the cmdq of the arm command
570  * @wqe: the wqe of the arm command
571  *
572  * Return 0 - Success, negative - Failure
573  **/
574 static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq,
575 				struct hinic_cmdq_wqe *wqe)
576 {
577 	struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
578 	struct hinic_cmdq_wqe_scmd *wqe_scmd;
579 	struct hinic_ctrl *ctrl;
580 	u32 ctrl_info;
581 
582 	wqe_scmd = &direct_wqe->wqe_scmd;
583 	ctrl = &wqe_scmd->ctrl;
584 	ctrl_info = be32_to_cpu(ctrl->ctrl_info);
585 
586 	/* HW should toggle the HW BUSY BIT */
587 	if (!CMDQ_WQE_COMPLETED(ctrl_info))
588 		return -EBUSY;
589 
590 	clear_wqe_complete_bit(cmdq, wqe);
591 
592 	hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE);
593 	return 0;
594 }
595 
596 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx,
597 				int errcode)
598 {
599 	if (cmdq->errcode[prod_idx])
600 		*cmdq->errcode[prod_idx] = errcode;
601 }
602 
603 /**
604  * cmdq_arm_ceq_handler - cmdq completion event handler for sync command
605  * @cmdq: the cmdq of the command
606  * @cons_idx: the consumer index to update the error code for
607  * @errcode: the error code
608  **/
609 static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx,
610 				  int errcode)
611 {
612 	u16 prod_idx = cons_idx;
613 
614 	spin_lock(&cmdq->cmdq_lock);
615 	cmdq_update_errcode(cmdq, prod_idx, errcode);
616 
617 	wmb();  /* write all before update for the command request */
618 
619 	if (cmdq->done[prod_idx])
620 		complete(cmdq->done[prod_idx]);
621 	spin_unlock(&cmdq->cmdq_lock);
622 }
623 
624 static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci,
625 				struct hinic_cmdq_wqe *cmdq_wqe)
626 {
627 	struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &cmdq_wqe->wqe_lcmd;
628 	struct hinic_status *status = &wqe_lcmd->status;
629 	struct hinic_ctrl *ctrl = &wqe_lcmd->ctrl;
630 	int errcode;
631 
632 	if (!CMDQ_WQE_COMPLETED(be32_to_cpu(ctrl->ctrl_info)))
633 		return -EBUSY;
634 
635 	errcode = CMDQ_WQE_ERRCODE_GET(be32_to_cpu(status->status_info), VAL);
636 
637 	cmdq_sync_cmd_handler(cmdq, ci, errcode);
638 
639 	clear_wqe_complete_bit(cmdq, cmdq_wqe);
640 	hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE);
641 	return 0;
642 }
643 
644 /**
645  * cmdq_ceq_handler - cmdq completion event handler
646  * @handle: private data for the handler(cmdqs)
647  * @ceqe_data: ceq element data
648  **/
649 static void cmdq_ceq_handler(void *handle, u32 ceqe_data)
650 {
651 	enum hinic_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE);
652 	struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)handle;
653 	struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type];
654 	struct hinic_cmdq_header *header;
655 	struct hinic_hw_wqe *hw_wqe;
656 	int err, set_arm = 0;
657 	u32 saved_data;
658 	u16 ci;
659 
660 	/* Read the smallest wqe size for getting wqe size */
661 	while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) {
662 		if (IS_ERR(hw_wqe))
663 			break;
664 
665 		header = CMDQ_WQE_HEADER(&hw_wqe->cmdq_wqe);
666 		saved_data = be32_to_cpu(header->saved_data);
667 
668 		if (HINIC_SAVED_DATA_GET(saved_data, ARM)) {
669 			/* arm_bit was set until here */
670 			set_arm = 0;
671 
672 			if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe))
673 				break;
674 		} else {
675 			set_arm = 1;
676 
677 			hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci);
678 			if (IS_ERR(hw_wqe))
679 				break;
680 
681 			if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe))
682 				break;
683 		}
684 	}
685 
686 	if (set_arm) {
687 		struct hinic_hwif *hwif = cmdqs->hwif;
688 		struct pci_dev *pdev = hwif->pdev;
689 
690 		err = hinic_set_arm_bit(cmdqs, HINIC_SET_ARM_CMDQ, cmdq_type);
691 		if (err)
692 			dev_err(&pdev->dev, "Failed to set arm for CMDQ\n");
693 	}
694 }
695 
696 /**
697  * cmdq_init_queue_ctxt - init the queue ctxt of a cmdq
698  * @cmdq_ctxt: cmdq ctxt to initialize
699  * @cmdq: the cmdq
700  * @cmdq_pages: the memory of the queue
701  **/
702 static void cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt *cmdq_ctxt,
703 				 struct hinic_cmdq *cmdq,
704 				 struct hinic_cmdq_pages *cmdq_pages)
705 {
706 	struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info;
707 	u64 wq_first_page_paddr, cmdq_first_block_paddr, pfn;
708 	struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq);
709 	struct hinic_wq *wq = cmdq->wq;
710 
711 	/* The data in the HW is in Big Endian Format */
712 	wq_first_page_paddr = be64_to_cpu(*wq->block_vaddr);
713 
714 	pfn = CMDQ_PFN(wq_first_page_paddr, wq->wq_page_size);
715 
716 	ctxt_info->curr_wqe_page_pfn =
717 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN)   |
718 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) |
719 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_ARM)               |
720 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN)                |
721 		HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED);
722 
723 	/* block PFN - Read Modify Write */
724 	cmdq_first_block_paddr = cmdq_pages->page_paddr;
725 
726 	pfn = CMDQ_PFN(cmdq_first_block_paddr, wq->wq_page_size);
727 
728 	ctxt_info->wq_block_pfn =
729 		HINIC_CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN) |
730 		HINIC_CMDQ_CTXT_BLOCK_INFO_SET(atomic_read(&wq->cons_idx), CI);
731 
732 	cmdq_ctxt->func_idx = HINIC_HWIF_FUNC_IDX(cmdqs->hwif);
733 	cmdq_ctxt->cmdq_type  = cmdq->cmdq_type;
734 }
735 
736 /**
737  * init_cmdq - initialize cmdq
738  * @cmdq: the cmdq
739  * @wq: the wq attaced to the cmdq
740  * @q_type: the cmdq type of the cmdq
741  * @db_area: doorbell area for the cmdq
742  *
743  * Return 0 - Success, negative - Failure
744  **/
745 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq,
746 		     enum hinic_cmdq_type q_type, void __iomem *db_area)
747 {
748 	int err;
749 
750 	cmdq->wq = wq;
751 	cmdq->cmdq_type = q_type;
752 	cmdq->wrapped = 1;
753 
754 	spin_lock_init(&cmdq->cmdq_lock);
755 
756 	cmdq->done = vzalloc(array_size(sizeof(*cmdq->done), wq->q_depth));
757 	if (!cmdq->done)
758 		return -ENOMEM;
759 
760 	cmdq->errcode = vzalloc(array_size(sizeof(*cmdq->errcode),
761 					   wq->q_depth));
762 	if (!cmdq->errcode) {
763 		err = -ENOMEM;
764 		goto err_errcode;
765 	}
766 
767 	cmdq->db_base = db_area + CMDQ_DB_OFF;
768 	return 0;
769 
770 err_errcode:
771 	vfree(cmdq->done);
772 	return err;
773 }
774 
775 /**
776  * free_cmdq - Free cmdq
777  * @cmdq: the cmdq to free
778  **/
779 static void free_cmdq(struct hinic_cmdq *cmdq)
780 {
781 	vfree(cmdq->errcode);
782 	vfree(cmdq->done);
783 }
784 
785 /**
786  * init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq
787  * @hwdev: the NIC HW device
788  * @cmdqs: cmdqs to write the ctxts for
789  * &db_area: db_area for all the cmdqs
790  *
791  * Return 0 - Success, negative - Failure
792  **/
793 static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev,
794 			   struct hinic_cmdqs *cmdqs, void __iomem **db_area)
795 {
796 	struct hinic_hwif *hwif = hwdev->hwif;
797 	enum hinic_cmdq_type type, cmdq_type;
798 	struct hinic_cmdq_ctxt *cmdq_ctxts;
799 	struct pci_dev *pdev = hwif->pdev;
800 	struct hinic_pfhwdev *pfhwdev;
801 	size_t cmdq_ctxts_size;
802 	int err;
803 
804 	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
805 		dev_err(&pdev->dev, "Unsupported PCI function type\n");
806 		return -EINVAL;
807 	}
808 
809 	cmdq_ctxts_size = HINIC_MAX_CMDQ_TYPES * sizeof(*cmdq_ctxts);
810 	cmdq_ctxts = devm_kzalloc(&pdev->dev, cmdq_ctxts_size, GFP_KERNEL);
811 	if (!cmdq_ctxts)
812 		return -ENOMEM;
813 
814 	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
815 
816 	cmdq_type = HINIC_CMDQ_SYNC;
817 	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
818 		err = init_cmdq(&cmdqs->cmdq[cmdq_type],
819 				&cmdqs->saved_wqs[cmdq_type], cmdq_type,
820 				db_area[cmdq_type]);
821 		if (err) {
822 			dev_err(&pdev->dev, "Failed to initialize cmdq\n");
823 			goto err_init_cmdq;
824 		}
825 
826 		cmdq_init_queue_ctxt(&cmdq_ctxts[cmdq_type],
827 				     &cmdqs->cmdq[cmdq_type],
828 				     &cmdqs->cmdq_pages);
829 	}
830 
831 	/* Write the CMDQ ctxts */
832 	cmdq_type = HINIC_CMDQ_SYNC;
833 	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
834 		err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
835 					HINIC_COMM_CMD_CMDQ_CTXT_SET,
836 					&cmdq_ctxts[cmdq_type],
837 					sizeof(cmdq_ctxts[cmdq_type]),
838 					NULL, NULL, HINIC_MGMT_MSG_SYNC);
839 		if (err) {
840 			dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n",
841 				cmdq_type);
842 			goto err_write_cmdq_ctxt;
843 		}
844 	}
845 
846 	devm_kfree(&pdev->dev, cmdq_ctxts);
847 	return 0;
848 
849 err_write_cmdq_ctxt:
850 	cmdq_type = HINIC_MAX_CMDQ_TYPES;
851 
852 err_init_cmdq:
853 	for (type = HINIC_CMDQ_SYNC; type < cmdq_type; type++)
854 		free_cmdq(&cmdqs->cmdq[type]);
855 
856 	devm_kfree(&pdev->dev, cmdq_ctxts);
857 	return err;
858 }
859 
860 /**
861  * hinic_init_cmdqs - init all cmdqs
862  * @cmdqs: cmdqs to init
863  * @hwif: HW interface for accessing cmdqs
864  * @db_area: doorbell areas for all the cmdqs
865  *
866  * Return 0 - Success, negative - Failure
867  **/
868 int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif,
869 		     void __iomem **db_area)
870 {
871 	struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs);
872 	struct pci_dev *pdev = hwif->pdev;
873 	struct hinic_hwdev *hwdev;
874 	size_t saved_wqs_size;
875 	u16 max_wqe_size;
876 	int err;
877 
878 	cmdqs->hwif = hwif;
879 	cmdqs->cmdq_buf_pool = dma_pool_create("hinic_cmdq", &pdev->dev,
880 					       HINIC_CMDQ_BUF_SIZE,
881 					       HINIC_CMDQ_BUF_SIZE, 0);
882 	if (!cmdqs->cmdq_buf_pool)
883 		return -ENOMEM;
884 
885 	saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq);
886 	cmdqs->saved_wqs = devm_kzalloc(&pdev->dev, saved_wqs_size, GFP_KERNEL);
887 	if (!cmdqs->saved_wqs) {
888 		err = -ENOMEM;
889 		goto err_saved_wqs;
890 	}
891 
892 	max_wqe_size = WQE_LCMD_SIZE;
893 	err = hinic_wqs_cmdq_alloc(&cmdqs->cmdq_pages, cmdqs->saved_wqs, hwif,
894 				   HINIC_MAX_CMDQ_TYPES, CMDQ_WQEBB_SIZE,
895 				   CMDQ_WQ_PAGE_SIZE, CMDQ_DEPTH, max_wqe_size);
896 	if (err) {
897 		dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n");
898 		goto err_cmdq_wqs;
899 	}
900 
901 	hwdev = container_of(func_to_io, struct hinic_hwdev, func_to_io);
902 	err = init_cmdqs_ctxt(hwdev, cmdqs, db_area);
903 	if (err) {
904 		dev_err(&pdev->dev, "Failed to write cmdq ctxt\n");
905 		goto err_cmdq_ctxt;
906 	}
907 
908 	hinic_ceq_register_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ, cmdqs,
909 			      cmdq_ceq_handler);
910 	return 0;
911 
912 err_cmdq_ctxt:
913 	hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs,
914 			    HINIC_MAX_CMDQ_TYPES);
915 
916 err_cmdq_wqs:
917 	devm_kfree(&pdev->dev, cmdqs->saved_wqs);
918 
919 err_saved_wqs:
920 	dma_pool_destroy(cmdqs->cmdq_buf_pool);
921 	return err;
922 }
923 
924 /**
925  * hinic_free_cmdqs - free all cmdqs
926  * @cmdqs: cmdqs to free
927  **/
928 void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs)
929 {
930 	struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs);
931 	struct hinic_hwif *hwif = cmdqs->hwif;
932 	struct pci_dev *pdev = hwif->pdev;
933 	enum hinic_cmdq_type cmdq_type;
934 
935 	hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ);
936 
937 	cmdq_type = HINIC_CMDQ_SYNC;
938 	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++)
939 		free_cmdq(&cmdqs->cmdq[cmdq_type]);
940 
941 	hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs,
942 			    HINIC_MAX_CMDQ_TYPES);
943 
944 	devm_kfree(&pdev->dev, cmdqs->saved_wqs);
945 
946 	dma_pool_destroy(cmdqs->cmdq_buf_pool);
947 }
948