1 /*
2  * Huawei HiNIC PCI Express Linux driver
3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12  * for more details.
13  *
14  */
15 
16 #include <linux/kernel.h>
17 #include <linux/types.h>
18 #include <linux/errno.h>
19 #include <linux/pci.h>
20 #include <linux/device.h>
21 #include <linux/slab.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/bitops.h>
24 #include <linux/err.h>
25 #include <linux/jiffies.h>
26 #include <linux/delay.h>
27 #include <linux/log2.h>
28 #include <linux/semaphore.h>
29 #include <asm/byteorder.h>
30 #include <asm/barrier.h>
31 
32 #include "hinic_hw_csr.h"
33 #include "hinic_hw_if.h"
34 #include "hinic_hw_api_cmd.h"
35 
36 #define API_CHAIN_NUM_CELLS                     32
37 
38 #define API_CMD_CELL_SIZE_SHIFT                 6
39 #define API_CMD_CELL_SIZE_MIN                   (BIT(API_CMD_CELL_SIZE_SHIFT))
40 
41 #define API_CMD_CELL_SIZE(cell_size)            \
42 		(((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \
43 		 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN)
44 
45 #define API_CMD_CELL_SIZE_VAL(size)             \
46 		ilog2((size) >> API_CMD_CELL_SIZE_SHIFT)
47 
48 #define API_CMD_BUF_SIZE                        2048
49 
50 /* Sizes of the members in hinic_api_cmd_cell */
51 #define API_CMD_CELL_DESC_SIZE          8
52 #define API_CMD_CELL_DATA_ADDR_SIZE     8
53 
54 #define API_CMD_CELL_ALIGNMENT          8
55 
56 #define API_CMD_TIMEOUT                 1000
57 
58 #define MASKED_IDX(chain, idx)          ((idx) & ((chain)->num_cells - 1))
59 
60 #define SIZE_8BYTES(size)               (ALIGN((size), 8) >> 3)
61 #define SIZE_4BYTES(size)               (ALIGN((size), 4) >> 2)
62 
63 #define RD_DMA_ATTR_DEFAULT             0
64 #define WR_DMA_ATTR_DEFAULT             0
65 
66 enum api_cmd_data_format {
67 	SGE_DATA = 1,           /* cell data is passed by hw address */
68 };
69 
70 enum api_cmd_type {
71 	API_CMD_WRITE = 0,
72 };
73 
74 enum api_cmd_bypass {
75 	NO_BYPASS       = 0,
76 	BYPASS          = 1,
77 };
78 
79 enum api_cmd_xor_chk_level {
80 	XOR_CHK_DIS = 0,
81 
82 	XOR_CHK_ALL = 3,
83 };
84 
85 static u8 xor_chksum_set(void *data)
86 {
87 	int idx;
88 	u8 *val, checksum = 0;
89 
90 	val = data;
91 
92 	for (idx = 0; idx < 7; idx++)
93 		checksum ^= val[idx];
94 
95 	return checksum;
96 }
97 
98 static void set_prod_idx(struct hinic_api_cmd_chain *chain)
99 {
100 	enum hinic_api_cmd_chain_type chain_type = chain->chain_type;
101 	struct hinic_hwif *hwif = chain->hwif;
102 	u32 addr, prod_idx;
103 
104 	addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type);
105 	prod_idx = hinic_hwif_read_reg(hwif, addr);
106 
107 	prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX);
108 
109 	prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX);
110 
111 	hinic_hwif_write_reg(hwif, addr, prod_idx);
112 }
113 
114 static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)
115 {
116 	u32 addr, val;
117 
118 	addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);
119 	val  = hinic_hwif_read_reg(chain->hwif, addr);
120 
121 	return HINIC_API_CMD_STATUS_GET(val, CONS_IDX);
122 }
123 
124 /**
125  * chain_busy - check if the chain is still processing last requests
126  * @chain: chain to check
127  *
128  * Return 0 - Success, negative - Failure
129  **/
130 static int chain_busy(struct hinic_api_cmd_chain *chain)
131 {
132 	struct hinic_hwif *hwif = chain->hwif;
133 	struct pci_dev *pdev = hwif->pdev;
134 	u32 prod_idx;
135 
136 	switch (chain->chain_type) {
137 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
138 		chain->cons_idx = get_hw_cons_idx(chain);
139 		prod_idx = chain->prod_idx;
140 
141 		/* check for a space for a new command */
142 		if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) {
143 			dev_err(&pdev->dev, "API CMD chain %d is busy\n",
144 				chain->chain_type);
145 			return -EBUSY;
146 		}
147 		break;
148 
149 	default:
150 		dev_err(&pdev->dev, "Unknown API CMD Chain type\n");
151 		break;
152 	}
153 
154 	return 0;
155 }
156 
157 /**
158  * get_cell_data_size - get the data size of a specific cell type
159  * @type: chain type
160  *
161  * Return the data(Desc + Address) size in the cell
162  **/
163 static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type)
164 {
165 	u8 cell_data_size = 0;
166 
167 	switch (type) {
168 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
169 		cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE +
170 				       API_CMD_CELL_DATA_ADDR_SIZE,
171 				       API_CMD_CELL_ALIGNMENT);
172 		break;
173 	default:
174 		break;
175 	}
176 
177 	return cell_data_size;
178 }
179 
180 /**
181  * prepare_cell_ctrl - prepare the ctrl of the cell for the command
182  * @cell_ctrl: the control of the cell to set the control value into it
183  * @data_size: the size of the data in the cell
184  **/
185 static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size)
186 {
187 	u8 chksum;
188 	u64 ctrl;
189 
190 	ctrl =  HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ)  |
191 		HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) |
192 		HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR);
193 
194 	chksum = xor_chksum_set(&ctrl);
195 
196 	ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM);
197 
198 	/* The data in the HW should be in Big Endian Format */
199 	*cell_ctrl = cpu_to_be64(ctrl);
200 }
201 
202 /**
203  * prepare_api_cmd - prepare API CMD command
204  * @chain: chain for the command
205  * @dest: destination node on the card that will receive the command
206  * @cmd: command data
207  * @cmd_size: the command size
208  **/
209 static void prepare_api_cmd(struct hinic_api_cmd_chain *chain,
210 			    enum hinic_node_id dest,
211 			    void *cmd, u16 cmd_size)
212 {
213 	struct hinic_api_cmd_cell *cell = chain->curr_node;
214 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
215 	struct hinic_hwif *hwif = chain->hwif;
216 	struct pci_dev *pdev = hwif->pdev;
217 
218 	cell_ctxt = &chain->cell_ctxt[chain->prod_idx];
219 
220 	switch (chain->chain_type) {
221 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
222 		cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE)   |
223 			     HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) |
224 			     HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS);
225 		break;
226 
227 	default:
228 		dev_err(&pdev->dev, "unknown Chain type\n");
229 		return;
230 	}
231 
232 	cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST)        |
233 		      HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE);
234 
235 	cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc),
236 					     XOR_CHKSUM);
237 
238 	/* The data in the HW should be in Big Endian Format */
239 	cell->desc = cpu_to_be64(cell->desc);
240 
241 	memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size);
242 }
243 
244 /**
245  * prepare_cell - prepare cell ctrl and cmd in the current cell
246  * @chain: chain for the command
247  * @dest: destination node on the card that will receive the command
248  * @cmd: command data
249  * @cmd_size: the command size
250  *
251  * Return 0 - Success, negative - Failure
252  **/
253 static void prepare_cell(struct hinic_api_cmd_chain *chain,
254 			 enum  hinic_node_id dest,
255 			 void *cmd, u16 cmd_size)
256 {
257 	struct hinic_api_cmd_cell *curr_node = chain->curr_node;
258 	u16 data_size = get_cell_data_size(chain->chain_type);
259 
260 	prepare_cell_ctrl(&curr_node->ctrl, data_size);
261 	prepare_api_cmd(chain, dest, cmd, cmd_size);
262 }
263 
264 static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain)
265 {
266 	chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1);
267 }
268 
269 /**
270  * api_cmd_status_update - update the status in the chain struct
271  * @chain: chain to update
272  **/
273 static void api_cmd_status_update(struct hinic_api_cmd_chain *chain)
274 {
275 	enum hinic_api_cmd_chain_type chain_type;
276 	struct hinic_api_cmd_status *wb_status;
277 	struct hinic_hwif *hwif = chain->hwif;
278 	struct pci_dev *pdev = hwif->pdev;
279 	u64 status_header;
280 	u32 status;
281 
282 	wb_status = chain->wb_status;
283 	status_header = be64_to_cpu(wb_status->header);
284 
285 	status = be32_to_cpu(wb_status->status);
286 	if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) {
287 		dev_err(&pdev->dev, "API CMD status: Xor check error\n");
288 		return;
289 	}
290 
291 	chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID);
292 	if (chain_type >= HINIC_API_CMD_MAX) {
293 		dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type);
294 		return;
295 	}
296 
297 	chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX);
298 }
299 
300 /**
301  * wait_for_status_poll - wait for write to api cmd command to complete
302  * @chain: the chain of the command
303  *
304  * Return 0 - Success, negative - Failure
305  **/
306 static int wait_for_status_poll(struct hinic_api_cmd_chain *chain)
307 {
308 	int err = -ETIMEDOUT;
309 	unsigned long end;
310 
311 	end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
312 	do {
313 		api_cmd_status_update(chain);
314 
315 		/* wait for CI to be updated - sign for completion */
316 		if (chain->cons_idx == chain->prod_idx) {
317 			err = 0;
318 			break;
319 		}
320 
321 		msleep(20);
322 	} while (time_before(jiffies, end));
323 
324 	return err;
325 }
326 
327 /**
328  * wait_for_api_cmd_completion - wait for command to complete
329  * @chain: chain for the command
330  *
331  * Return 0 - Success, negative - Failure
332  **/
333 static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain)
334 {
335 	struct hinic_hwif *hwif = chain->hwif;
336 	struct pci_dev *pdev = hwif->pdev;
337 	int err;
338 
339 	switch (chain->chain_type) {
340 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
341 		err = wait_for_status_poll(chain);
342 		if (err) {
343 			dev_err(&pdev->dev, "API CMD Poll status timeout\n");
344 			break;
345 		}
346 		break;
347 
348 	default:
349 		dev_err(&pdev->dev, "unknown API CMD Chain type\n");
350 		err = -EINVAL;
351 		break;
352 	}
353 
354 	return err;
355 }
356 
357 /**
358  * api_cmd - API CMD command
359  * @chain: chain for the command
360  * @dest: destination node on the card that will receive the command
361  * @cmd: command data
362  * @size: the command size
363  *
364  * Return 0 - Success, negative - Failure
365  **/
366 static int api_cmd(struct hinic_api_cmd_chain *chain,
367 		   enum hinic_node_id dest, u8 *cmd, u16 cmd_size)
368 {
369 	struct hinic_api_cmd_cell_ctxt *ctxt;
370 	int err;
371 
372 	down(&chain->sem);
373 	if (chain_busy(chain)) {
374 		up(&chain->sem);
375 		return -EBUSY;
376 	}
377 
378 	prepare_cell(chain, dest, cmd, cmd_size);
379 	cmd_chain_prod_idx_inc(chain);
380 
381 	wmb();  /* inc pi before issue the command */
382 
383 	set_prod_idx(chain);    /* issue the command */
384 
385 	ctxt = &chain->cell_ctxt[chain->prod_idx];
386 
387 	chain->curr_node = ctxt->cell_vaddr;
388 
389 	err = wait_for_api_cmd_completion(chain);
390 
391 	up(&chain->sem);
392 	return err;
393 }
394 
395 /**
396  * hinic_api_cmd_write - Write API CMD command
397  * @chain: chain for write command
398  * @dest: destination node on the card that will receive the command
399  * @cmd: command data
400  * @size: the command size
401  *
402  * Return 0 - Success, negative - Failure
403  **/
404 int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,
405 			enum hinic_node_id dest, u8 *cmd, u16 size)
406 {
407 	/* Verify the chain type */
408 	if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU)
409 		return api_cmd(chain, dest, cmd, size);
410 
411 	return -EINVAL;
412 }
413 
414 /**
415  * api_cmd_hw_restart - restart the chain in the HW
416  * @chain: the API CMD specific chain to restart
417  *
418  * Return 0 - Success, negative - Failure
419  **/
420 static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain)
421 {
422 	struct hinic_hwif *hwif = chain->hwif;
423 	int err = -ETIMEDOUT;
424 	unsigned long end;
425 	u32 reg_addr, val;
426 
427 	/* Read Modify Write */
428 	reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type);
429 	val = hinic_hwif_read_reg(hwif, reg_addr);
430 
431 	val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART);
432 	val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART);
433 
434 	hinic_hwif_write_reg(hwif, reg_addr, val);
435 
436 	end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
437 	do {
438 		val = hinic_hwif_read_reg(hwif, reg_addr);
439 
440 		if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) {
441 			err = 0;
442 			break;
443 		}
444 
445 		msleep(20);
446 	} while (time_before(jiffies, end));
447 
448 	return err;
449 }
450 
451 /**
452  * api_cmd_ctrl_init - set the control register of a chain
453  * @chain: the API CMD specific chain to set control register for
454  **/
455 static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain)
456 {
457 	struct hinic_hwif *hwif = chain->hwif;
458 	u32 addr, ctrl;
459 	u16 cell_size;
460 
461 	/* Read Modify Write */
462 	addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
463 
464 	cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size);
465 
466 	ctrl = hinic_hwif_read_reg(hwif, addr);
467 
468 	ctrl =  HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
469 		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR)         &
470 		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN)         &
471 		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN)      &
472 		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
473 
474 	ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR)              |
475 		HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) |
476 		HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE);
477 
478 	hinic_hwif_write_reg(hwif, addr, ctrl);
479 }
480 
481 /**
482  * api_cmd_set_status_addr - set the status address of a chain in the HW
483  * @chain: the API CMD specific chain to set in HW status address for
484  **/
485 static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain)
486 {
487 	struct hinic_hwif *hwif = chain->hwif;
488 	u32 addr, val;
489 
490 	addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type);
491 	val = upper_32_bits(chain->wb_status_paddr);
492 	hinic_hwif_write_reg(hwif, addr, val);
493 
494 	addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type);
495 	val = lower_32_bits(chain->wb_status_paddr);
496 	hinic_hwif_write_reg(hwif, addr, val);
497 }
498 
499 /**
500  * api_cmd_set_num_cells - set the number cells of a chain in the HW
501  * @chain: the API CMD specific chain to set in HW the number of cells for
502  **/
503 static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain)
504 {
505 	struct hinic_hwif *hwif = chain->hwif;
506 	u32 addr, val;
507 
508 	addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type);
509 	val  = chain->num_cells;
510 	hinic_hwif_write_reg(hwif, addr, val);
511 }
512 
513 /**
514  * api_cmd_head_init - set the head of a chain in the HW
515  * @chain: the API CMD specific chain to set in HW the head for
516  **/
517 static void api_cmd_head_init(struct hinic_api_cmd_chain *chain)
518 {
519 	struct hinic_hwif *hwif = chain->hwif;
520 	u32 addr, val;
521 
522 	addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type);
523 	val = upper_32_bits(chain->head_cell_paddr);
524 	hinic_hwif_write_reg(hwif, addr, val);
525 
526 	addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type);
527 	val = lower_32_bits(chain->head_cell_paddr);
528 	hinic_hwif_write_reg(hwif, addr, val);
529 }
530 
531 /**
532  * api_cmd_chain_hw_clean - clean the HW
533  * @chain: the API CMD specific chain
534  **/
535 static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain)
536 {
537 	struct hinic_hwif *hwif = chain->hwif;
538 	u32 addr, ctrl;
539 
540 	addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
541 
542 	ctrl = hinic_hwif_read_reg(hwif, addr);
543 	ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
544 	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR)         &
545 	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN)         &
546 	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN)      &
547 	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
548 
549 	hinic_hwif_write_reg(hwif, addr, ctrl);
550 }
551 
552 /**
553  * api_cmd_chain_hw_init - initialize the chain in the HW
554  * @chain: the API CMD specific chain to initialize in HW
555  *
556  * Return 0 - Success, negative - Failure
557  **/
558 static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain)
559 {
560 	struct hinic_hwif *hwif = chain->hwif;
561 	struct pci_dev *pdev = hwif->pdev;
562 	int err;
563 
564 	api_cmd_chain_hw_clean(chain);
565 
566 	api_cmd_set_status_addr(chain);
567 
568 	err = api_cmd_hw_restart(chain);
569 	if (err) {
570 		dev_err(&pdev->dev, "Failed to restart API CMD HW\n");
571 		return err;
572 	}
573 
574 	api_cmd_ctrl_init(chain);
575 	api_cmd_set_num_cells(chain);
576 	api_cmd_head_init(chain);
577 	return 0;
578 }
579 
580 /**
581  * free_cmd_buf - free the dma buffer of API CMD command
582  * @chain: the API CMD specific chain of the cmd
583  * @cell_idx: the cell index of the cmd
584  **/
585 static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx)
586 {
587 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
588 	struct hinic_hwif *hwif = chain->hwif;
589 	struct pci_dev *pdev = hwif->pdev;
590 
591 	cell_ctxt = &chain->cell_ctxt[cell_idx];
592 
593 	dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE,
594 			  cell_ctxt->api_cmd_vaddr,
595 			  cell_ctxt->api_cmd_paddr);
596 }
597 
598 /**
599  * alloc_cmd_buf - allocate a dma buffer for API CMD command
600  * @chain: the API CMD specific chain for the cmd
601  * @cell: the cell in the HW for the cmd
602  * @cell_idx: the index of the cell
603  *
604  * Return 0 - Success, negative - Failure
605  **/
606 static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
607 			 struct hinic_api_cmd_cell *cell, int cell_idx)
608 {
609 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
610 	struct hinic_hwif *hwif = chain->hwif;
611 	struct pci_dev *pdev = hwif->pdev;
612 	dma_addr_t cmd_paddr;
613 	u8 *cmd_vaddr;
614 	int err = 0;
615 
616 	cmd_vaddr = dma_zalloc_coherent(&pdev->dev, API_CMD_BUF_SIZE,
617 					&cmd_paddr, GFP_KERNEL);
618 	if (!cmd_vaddr) {
619 		dev_err(&pdev->dev, "Failed to allocate API CMD DMA memory\n");
620 		return -ENOMEM;
621 	}
622 
623 	cell_ctxt = &chain->cell_ctxt[cell_idx];
624 
625 	cell_ctxt->api_cmd_vaddr = cmd_vaddr;
626 	cell_ctxt->api_cmd_paddr = cmd_paddr;
627 
628 	/* set the cmd DMA address in the cell */
629 	switch (chain->chain_type) {
630 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
631 		/* The data in the HW should be in Big Endian Format */
632 		cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr);
633 		break;
634 
635 	default:
636 		dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
637 		free_cmd_buf(chain, cell_idx);
638 		err = -EINVAL;
639 		break;
640 	}
641 
642 	return err;
643 }
644 
645 /**
646  * api_cmd_create_cell - create API CMD cell for specific chain
647  * @chain: the API CMD specific chain to create its cell
648  * @cell_idx: the index of the cell to create
649  * @pre_node: previous cell
650  * @node_vaddr: the returned virt addr of the cell
651  *
652  * Return 0 - Success, negative - Failure
653  **/
654 static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
655 			       int cell_idx,
656 			       struct hinic_api_cmd_cell *pre_node,
657 			       struct hinic_api_cmd_cell **node_vaddr)
658 {
659 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
660 	struct hinic_hwif *hwif = chain->hwif;
661 	struct pci_dev *pdev = hwif->pdev;
662 	struct hinic_api_cmd_cell *node;
663 	dma_addr_t node_paddr;
664 	int err;
665 
666 	node = dma_zalloc_coherent(&pdev->dev, chain->cell_size,
667 				   &node_paddr, GFP_KERNEL);
668 	if (!node) {
669 		dev_err(&pdev->dev, "Failed to allocate dma API CMD cell\n");
670 		return -ENOMEM;
671 	}
672 
673 	node->read.hw_wb_resp_paddr = 0;
674 
675 	cell_ctxt = &chain->cell_ctxt[cell_idx];
676 	cell_ctxt->cell_vaddr = node;
677 	cell_ctxt->cell_paddr = node_paddr;
678 
679 	if (!pre_node) {
680 		chain->head_cell_paddr = node_paddr;
681 		chain->head_node = node;
682 	} else {
683 		/* The data in the HW should be in Big Endian Format */
684 		pre_node->next_cell_paddr = cpu_to_be64(node_paddr);
685 	}
686 
687 	switch (chain->chain_type) {
688 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
689 		err = alloc_cmd_buf(chain, node, cell_idx);
690 		if (err) {
691 			dev_err(&pdev->dev, "Failed to allocate cmd buffer\n");
692 			goto err_alloc_cmd_buf;
693 		}
694 		break;
695 
696 	default:
697 		dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
698 		err = -EINVAL;
699 		goto err_alloc_cmd_buf;
700 	}
701 
702 	*node_vaddr = node;
703 	return 0;
704 
705 err_alloc_cmd_buf:
706 	dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr);
707 	return err;
708 }
709 
710 /**
711  * api_cmd_destroy_cell - destroy API CMD cell of specific chain
712  * @chain: the API CMD specific chain to destroy its cell
713  * @cell_idx: the cell to destroy
714  **/
715 static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain,
716 				 int cell_idx)
717 {
718 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
719 	struct hinic_hwif *hwif = chain->hwif;
720 	struct pci_dev *pdev = hwif->pdev;
721 	struct hinic_api_cmd_cell *node;
722 	dma_addr_t node_paddr;
723 	size_t node_size;
724 
725 	cell_ctxt = &chain->cell_ctxt[cell_idx];
726 
727 	node = cell_ctxt->cell_vaddr;
728 	node_paddr = cell_ctxt->cell_paddr;
729 	node_size = chain->cell_size;
730 
731 	if (cell_ctxt->api_cmd_vaddr) {
732 		switch (chain->chain_type) {
733 		case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
734 			free_cmd_buf(chain, cell_idx);
735 			break;
736 		default:
737 			dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
738 			break;
739 		}
740 
741 		dma_free_coherent(&pdev->dev, node_size, node,
742 				  node_paddr);
743 	}
744 }
745 
746 /**
747  * api_cmd_destroy_cells - destroy API CMD cells of specific chain
748  * @chain: the API CMD specific chain to destroy its cells
749  * @num_cells: number of cells to destroy
750  **/
751 static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain,
752 				  int num_cells)
753 {
754 	int cell_idx;
755 
756 	for (cell_idx = 0; cell_idx < num_cells; cell_idx++)
757 		api_cmd_destroy_cell(chain, cell_idx);
758 }
759 
760 /**
761  * api_cmd_create_cells - create API CMD cells for specific chain
762  * @chain: the API CMD specific chain
763  *
764  * Return 0 - Success, negative - Failure
765  **/
766 static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain)
767 {
768 	struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL;
769 	struct hinic_hwif *hwif = chain->hwif;
770 	struct pci_dev *pdev = hwif->pdev;
771 	int err, cell_idx;
772 
773 	for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) {
774 		err = api_cmd_create_cell(chain, cell_idx, pre_node, &node);
775 		if (err) {
776 			dev_err(&pdev->dev, "Failed to create API CMD cell\n");
777 			goto err_create_cell;
778 		}
779 
780 		pre_node = node;
781 	}
782 
783 	/* set the Final node to point on the start */
784 	node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr);
785 
786 	/* set the current node to be the head */
787 	chain->curr_node = chain->head_node;
788 	return 0;
789 
790 err_create_cell:
791 	api_cmd_destroy_cells(chain, cell_idx);
792 	return err;
793 }
794 
795 /**
796  * api_chain_init - initialize API CMD specific chain
797  * @chain: the API CMD specific chain to initialize
798  * @attr: attributes to set in the chain
799  *
800  * Return 0 - Success, negative - Failure
801  **/
802 static int api_chain_init(struct hinic_api_cmd_chain *chain,
803 			  struct hinic_api_cmd_chain_attr *attr)
804 {
805 	struct hinic_hwif *hwif = attr->hwif;
806 	struct pci_dev *pdev = hwif->pdev;
807 	size_t cell_ctxt_size;
808 
809 	chain->hwif = hwif;
810 	chain->chain_type  = attr->chain_type;
811 	chain->num_cells = attr->num_cells;
812 	chain->cell_size = attr->cell_size;
813 
814 	chain->prod_idx  = 0;
815 	chain->cons_idx  = 0;
816 
817 	sema_init(&chain->sem, 1);
818 
819 	cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt);
820 	chain->cell_ctxt = devm_kzalloc(&pdev->dev, cell_ctxt_size, GFP_KERNEL);
821 	if (!chain->cell_ctxt)
822 		return -ENOMEM;
823 
824 	chain->wb_status = dma_zalloc_coherent(&pdev->dev,
825 					       sizeof(*chain->wb_status),
826 					       &chain->wb_status_paddr,
827 					       GFP_KERNEL);
828 	if (!chain->wb_status) {
829 		dev_err(&pdev->dev, "Failed to allocate DMA wb status\n");
830 		return -ENOMEM;
831 	}
832 
833 	return 0;
834 }
835 
836 /**
837  * api_chain_free - free API CMD specific chain
838  * @chain: the API CMD specific chain to free
839  **/
840 static void api_chain_free(struct hinic_api_cmd_chain *chain)
841 {
842 	struct hinic_hwif *hwif = chain->hwif;
843 	struct pci_dev *pdev = hwif->pdev;
844 
845 	dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status),
846 			  chain->wb_status, chain->wb_status_paddr);
847 }
848 
849 /**
850  * api_cmd_create_chain - create API CMD specific chain
851  * @attr: attributes to set the chain
852  *
853  * Return the created chain
854  **/
855 static struct hinic_api_cmd_chain *
856 	api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr)
857 {
858 	struct hinic_hwif *hwif = attr->hwif;
859 	struct pci_dev *pdev = hwif->pdev;
860 	struct hinic_api_cmd_chain *chain;
861 	int err;
862 
863 	if (attr->num_cells & (attr->num_cells - 1)) {
864 		dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n");
865 		return ERR_PTR(-EINVAL);
866 	}
867 
868 	chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
869 	if (!chain)
870 		return ERR_PTR(-ENOMEM);
871 
872 	err = api_chain_init(chain, attr);
873 	if (err) {
874 		dev_err(&pdev->dev, "Failed to initialize chain\n");
875 		return ERR_PTR(err);
876 	}
877 
878 	err = api_cmd_create_cells(chain);
879 	if (err) {
880 		dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n");
881 		goto err_create_cells;
882 	}
883 
884 	err = api_cmd_chain_hw_init(chain);
885 	if (err) {
886 		dev_err(&pdev->dev, "Failed to initialize chain HW\n");
887 		goto err_chain_hw_init;
888 	}
889 
890 	return chain;
891 
892 err_chain_hw_init:
893 	api_cmd_destroy_cells(chain, chain->num_cells);
894 
895 err_create_cells:
896 	api_chain_free(chain);
897 	return ERR_PTR(err);
898 }
899 
900 /**
901  * api_cmd_destroy_chain - destroy API CMD specific chain
902  * @chain: the API CMD specific chain to destroy
903  **/
904 static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain)
905 {
906 	api_cmd_chain_hw_clean(chain);
907 	api_cmd_destroy_cells(chain, chain->num_cells);
908 	api_chain_free(chain);
909 }
910 
911 /**
912  * hinic_api_cmd_init - Initialize all the API CMD chains
913  * @chain: the API CMD chains that are initialized
914  * @hwif: the hardware interface of a pci function device
915  *
916  * Return 0 - Success, negative - Failure
917  **/
918 int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain,
919 		       struct hinic_hwif *hwif)
920 {
921 	enum hinic_api_cmd_chain_type type, chain_type;
922 	struct hinic_api_cmd_chain_attr attr;
923 	struct pci_dev *pdev = hwif->pdev;
924 	size_t hw_cell_sz;
925 	int err;
926 
927 	hw_cell_sz = sizeof(struct hinic_api_cmd_cell);
928 
929 	attr.hwif = hwif;
930 	attr.num_cells  = API_CHAIN_NUM_CELLS;
931 	attr.cell_size  = API_CMD_CELL_SIZE(hw_cell_sz);
932 
933 	chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
934 	for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
935 		attr.chain_type = chain_type;
936 
937 		if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
938 			continue;
939 
940 		chain[chain_type] = api_cmd_create_chain(&attr);
941 		if (IS_ERR(chain[chain_type])) {
942 			dev_err(&pdev->dev, "Failed to create chain %d\n",
943 				chain_type);
944 			err = PTR_ERR(chain[chain_type]);
945 			goto err_create_chain;
946 		}
947 	}
948 
949 	return 0;
950 
951 err_create_chain:
952 	type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
953 	for ( ; type < chain_type; type++) {
954 		if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
955 			continue;
956 
957 		api_cmd_destroy_chain(chain[type]);
958 	}
959 
960 	return err;
961 }
962 
963 /**
964  * hinic_api_cmd_free - free the API CMD chains
965  * @chain: the API CMD chains that are freed
966  **/
967 void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain)
968 {
969 	enum hinic_api_cmd_chain_type chain_type;
970 
971 	chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
972 	for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
973 		if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
974 			continue;
975 
976 		api_cmd_destroy_chain(chain[chain_type]);
977 	}
978 }
979