1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Huawei HiNIC PCI Express Linux driver
4  * Copyright(c) 2017 Huawei Technologies Co., Ltd
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/errno.h>
10 #include <linux/pci.h>
11 #include <linux/device.h>
12 #include <linux/slab.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/bitops.h>
15 #include <linux/err.h>
16 #include <linux/jiffies.h>
17 #include <linux/delay.h>
18 #include <linux/log2.h>
19 #include <linux/semaphore.h>
20 #include <asm/byteorder.h>
21 #include <asm/barrier.h>
22 
23 #include "hinic_hw_csr.h"
24 #include "hinic_hw_if.h"
25 #include "hinic_hw_api_cmd.h"
26 
27 #define API_CHAIN_NUM_CELLS                     32
28 
29 #define API_CMD_CELL_SIZE_SHIFT                 6
30 #define API_CMD_CELL_SIZE_MIN                   (BIT(API_CMD_CELL_SIZE_SHIFT))
31 
32 #define API_CMD_CELL_SIZE(cell_size)            \
33 		(((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \
34 		 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN)
35 
36 #define API_CMD_CELL_SIZE_VAL(size)             \
37 		ilog2((size) >> API_CMD_CELL_SIZE_SHIFT)
38 
39 #define API_CMD_BUF_SIZE                        2048
40 
41 /* Sizes of the members in hinic_api_cmd_cell */
42 #define API_CMD_CELL_DESC_SIZE          8
43 #define API_CMD_CELL_DATA_ADDR_SIZE     8
44 
45 #define API_CMD_CELL_ALIGNMENT          8
46 
47 #define API_CMD_TIMEOUT                 1000
48 
49 #define MASKED_IDX(chain, idx)          ((idx) & ((chain)->num_cells - 1))
50 
51 #define SIZE_8BYTES(size)               (ALIGN((size), 8) >> 3)
52 #define SIZE_4BYTES(size)               (ALIGN((size), 4) >> 2)
53 
54 #define RD_DMA_ATTR_DEFAULT             0
55 #define WR_DMA_ATTR_DEFAULT             0
56 
57 enum api_cmd_data_format {
58 	SGE_DATA = 1,           /* cell data is passed by hw address */
59 };
60 
61 enum api_cmd_type {
62 	API_CMD_WRITE = 0,
63 };
64 
65 enum api_cmd_bypass {
66 	NO_BYPASS       = 0,
67 	BYPASS          = 1,
68 };
69 
70 enum api_cmd_xor_chk_level {
71 	XOR_CHK_DIS = 0,
72 
73 	XOR_CHK_ALL = 3,
74 };
75 
76 static u8 xor_chksum_set(void *data)
77 {
78 	int idx;
79 	u8 *val, checksum = 0;
80 
81 	val = data;
82 
83 	for (idx = 0; idx < 7; idx++)
84 		checksum ^= val[idx];
85 
86 	return checksum;
87 }
88 
89 static void set_prod_idx(struct hinic_api_cmd_chain *chain)
90 {
91 	enum hinic_api_cmd_chain_type chain_type = chain->chain_type;
92 	struct hinic_hwif *hwif = chain->hwif;
93 	u32 addr, prod_idx;
94 
95 	addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type);
96 	prod_idx = hinic_hwif_read_reg(hwif, addr);
97 
98 	prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX);
99 
100 	prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX);
101 
102 	hinic_hwif_write_reg(hwif, addr, prod_idx);
103 }
104 
105 static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)
106 {
107 	u32 addr, val;
108 
109 	addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);
110 	val  = hinic_hwif_read_reg(chain->hwif, addr);
111 
112 	return HINIC_API_CMD_STATUS_GET(val, CONS_IDX);
113 }
114 
115 static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain)
116 {
117 	u32 addr, val;
118 
119 	addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);
120 	val  = hinic_hwif_read_reg(chain->hwif, addr);
121 
122 	dev_err(&chain->hwif->pdev->dev, "Chain type: 0x%x, cpld error: 0x%x, check error: 0x%x, current fsm: 0x%x\n",
123 		chain->chain_type, HINIC_API_CMD_STATUS_GET(val, CPLD_ERR),
124 		HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR),
125 		HINIC_API_CMD_STATUS_GET(val, FSM));
126 
127 	dev_err(&chain->hwif->pdev->dev, "Chain hw current ci: 0x%x\n",
128 		HINIC_API_CMD_STATUS_GET(val, CONS_IDX));
129 
130 	addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type);
131 	val  = hinic_hwif_read_reg(chain->hwif, addr);
132 	dev_err(&chain->hwif->pdev->dev, "Chain hw current pi: 0x%x\n", val);
133 }
134 
135 /**
136  * chain_busy - check if the chain is still processing last requests
137  * @chain: chain to check
138  *
139  * Return 0 - Success, negative - Failure
140  **/
141 static int chain_busy(struct hinic_api_cmd_chain *chain)
142 {
143 	struct hinic_hwif *hwif = chain->hwif;
144 	struct pci_dev *pdev = hwif->pdev;
145 	u32 prod_idx;
146 
147 	switch (chain->chain_type) {
148 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
149 		chain->cons_idx = get_hw_cons_idx(chain);
150 		prod_idx = chain->prod_idx;
151 
152 		/* check for a space for a new command */
153 		if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) {
154 			dev_err(&pdev->dev, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d\n",
155 				chain->chain_type, chain->cons_idx,
156 				chain->prod_idx);
157 			dump_api_chain_reg(chain);
158 			return -EBUSY;
159 		}
160 		break;
161 
162 	default:
163 		dev_err(&pdev->dev, "Unknown API CMD Chain type\n");
164 		break;
165 	}
166 
167 	return 0;
168 }
169 
170 /**
171  * get_cell_data_size - get the data size of a specific cell type
172  * @type: chain type
173  *
174  * Return the data(Desc + Address) size in the cell
175  **/
176 static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type)
177 {
178 	u8 cell_data_size = 0;
179 
180 	switch (type) {
181 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
182 		cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE +
183 				       API_CMD_CELL_DATA_ADDR_SIZE,
184 				       API_CMD_CELL_ALIGNMENT);
185 		break;
186 	default:
187 		break;
188 	}
189 
190 	return cell_data_size;
191 }
192 
193 /**
194  * prepare_cell_ctrl - prepare the ctrl of the cell for the command
195  * @cell_ctrl: the control of the cell to set the control value into it
196  * @data_size: the size of the data in the cell
197  **/
198 static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size)
199 {
200 	u8 chksum;
201 	u64 ctrl;
202 
203 	ctrl =  HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ)  |
204 		HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) |
205 		HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR);
206 
207 	chksum = xor_chksum_set(&ctrl);
208 
209 	ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM);
210 
211 	/* The data in the HW should be in Big Endian Format */
212 	*cell_ctrl = cpu_to_be64(ctrl);
213 }
214 
215 /**
216  * prepare_api_cmd - prepare API CMD command
217  * @chain: chain for the command
218  * @dest: destination node on the card that will receive the command
219  * @cmd: command data
220  * @cmd_size: the command size
221  **/
222 static void prepare_api_cmd(struct hinic_api_cmd_chain *chain,
223 			    enum hinic_node_id dest,
224 			    void *cmd, u16 cmd_size)
225 {
226 	struct hinic_api_cmd_cell *cell = chain->curr_node;
227 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
228 	struct hinic_hwif *hwif = chain->hwif;
229 	struct pci_dev *pdev = hwif->pdev;
230 
231 	cell_ctxt = &chain->cell_ctxt[chain->prod_idx];
232 
233 	switch (chain->chain_type) {
234 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
235 		cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE)   |
236 			     HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) |
237 			     HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS);
238 		break;
239 
240 	default:
241 		dev_err(&pdev->dev, "unknown Chain type\n");
242 		return;
243 	}
244 
245 	cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST)        |
246 		      HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE);
247 
248 	cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc),
249 					     XOR_CHKSUM);
250 
251 	/* The data in the HW should be in Big Endian Format */
252 	cell->desc = cpu_to_be64(cell->desc);
253 
254 	memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size);
255 }
256 
257 /**
258  * prepare_cell - prepare cell ctrl and cmd in the current cell
259  * @chain: chain for the command
260  * @dest: destination node on the card that will receive the command
261  * @cmd: command data
262  * @cmd_size: the command size
263  *
264  * Return 0 - Success, negative - Failure
265  **/
266 static void prepare_cell(struct hinic_api_cmd_chain *chain,
267 			 enum  hinic_node_id dest,
268 			 void *cmd, u16 cmd_size)
269 {
270 	struct hinic_api_cmd_cell *curr_node = chain->curr_node;
271 	u16 data_size = get_cell_data_size(chain->chain_type);
272 
273 	prepare_cell_ctrl(&curr_node->ctrl, data_size);
274 	prepare_api_cmd(chain, dest, cmd, cmd_size);
275 }
276 
277 static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain)
278 {
279 	chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1);
280 }
281 
282 /**
283  * api_cmd_status_update - update the status in the chain struct
284  * @chain: chain to update
285  **/
286 static void api_cmd_status_update(struct hinic_api_cmd_chain *chain)
287 {
288 	enum hinic_api_cmd_chain_type chain_type;
289 	struct hinic_api_cmd_status *wb_status;
290 	struct hinic_hwif *hwif = chain->hwif;
291 	struct pci_dev *pdev = hwif->pdev;
292 	u64 status_header;
293 	u32 status;
294 
295 	wb_status = chain->wb_status;
296 	status_header = be64_to_cpu(wb_status->header);
297 
298 	status = be32_to_cpu(wb_status->status);
299 	if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) {
300 		dev_err(&pdev->dev, "API CMD status: Xor check error\n");
301 		return;
302 	}
303 
304 	chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID);
305 	if (chain_type >= HINIC_API_CMD_MAX) {
306 		dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type);
307 		return;
308 	}
309 
310 	chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX);
311 }
312 
313 /**
314  * wait_for_status_poll - wait for write to api cmd command to complete
315  * @chain: the chain of the command
316  *
317  * Return 0 - Success, negative - Failure
318  **/
319 static int wait_for_status_poll(struct hinic_api_cmd_chain *chain)
320 {
321 	int err = -ETIMEDOUT;
322 	unsigned long end;
323 
324 	end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
325 	do {
326 		api_cmd_status_update(chain);
327 
328 		/* wait for CI to be updated - sign for completion */
329 		if (chain->cons_idx == chain->prod_idx) {
330 			err = 0;
331 			break;
332 		}
333 
334 		msleep(20);
335 	} while (time_before(jiffies, end));
336 
337 	return err;
338 }
339 
340 /**
341  * wait_for_api_cmd_completion - wait for command to complete
342  * @chain: chain for the command
343  *
344  * Return 0 - Success, negative - Failure
345  **/
346 static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain)
347 {
348 	struct hinic_hwif *hwif = chain->hwif;
349 	struct pci_dev *pdev = hwif->pdev;
350 	int err;
351 
352 	switch (chain->chain_type) {
353 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
354 		err = wait_for_status_poll(chain);
355 		if (err) {
356 			dev_err(&pdev->dev, "API CMD Poll status timeout\n");
357 			dump_api_chain_reg(chain);
358 			break;
359 		}
360 		break;
361 
362 	default:
363 		dev_err(&pdev->dev, "unknown API CMD Chain type\n");
364 		err = -EINVAL;
365 		break;
366 	}
367 
368 	return err;
369 }
370 
371 /**
372  * api_cmd - API CMD command
373  * @chain: chain for the command
374  * @dest: destination node on the card that will receive the command
375  * @cmd: command data
376  * @cmd_size: the command size
377  *
378  * Return 0 - Success, negative - Failure
379  **/
380 static int api_cmd(struct hinic_api_cmd_chain *chain,
381 		   enum hinic_node_id dest, u8 *cmd, u16 cmd_size)
382 {
383 	struct hinic_api_cmd_cell_ctxt *ctxt;
384 	int err;
385 
386 	down(&chain->sem);
387 	if (chain_busy(chain)) {
388 		up(&chain->sem);
389 		return -EBUSY;
390 	}
391 
392 	prepare_cell(chain, dest, cmd, cmd_size);
393 	cmd_chain_prod_idx_inc(chain);
394 
395 	wmb();  /* inc pi before issue the command */
396 
397 	set_prod_idx(chain);    /* issue the command */
398 
399 	ctxt = &chain->cell_ctxt[chain->prod_idx];
400 
401 	chain->curr_node = ctxt->cell_vaddr;
402 
403 	err = wait_for_api_cmd_completion(chain);
404 
405 	up(&chain->sem);
406 	return err;
407 }
408 
409 /**
410  * hinic_api_cmd_write - Write API CMD command
411  * @chain: chain for write command
412  * @dest: destination node on the card that will receive the command
413  * @cmd: command data
414  * @size: the command size
415  *
416  * Return 0 - Success, negative - Failure
417  **/
418 int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,
419 			enum hinic_node_id dest, u8 *cmd, u16 size)
420 {
421 	/* Verify the chain type */
422 	if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU)
423 		return api_cmd(chain, dest, cmd, size);
424 
425 	return -EINVAL;
426 }
427 
428 /**
429  * api_cmd_hw_restart - restart the chain in the HW
430  * @chain: the API CMD specific chain to restart
431  *
432  * Return 0 - Success, negative - Failure
433  **/
434 static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain)
435 {
436 	struct hinic_hwif *hwif = chain->hwif;
437 	int err = -ETIMEDOUT;
438 	unsigned long end;
439 	u32 reg_addr, val;
440 
441 	/* Read Modify Write */
442 	reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type);
443 	val = hinic_hwif_read_reg(hwif, reg_addr);
444 
445 	val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART);
446 	val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART);
447 
448 	hinic_hwif_write_reg(hwif, reg_addr, val);
449 
450 	end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
451 	do {
452 		val = hinic_hwif_read_reg(hwif, reg_addr);
453 
454 		if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) {
455 			err = 0;
456 			break;
457 		}
458 
459 		msleep(20);
460 	} while (time_before(jiffies, end));
461 
462 	return err;
463 }
464 
465 /**
466  * api_cmd_ctrl_init - set the control register of a chain
467  * @chain: the API CMD specific chain to set control register for
468  **/
469 static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain)
470 {
471 	struct hinic_hwif *hwif = chain->hwif;
472 	u32 addr, ctrl;
473 	u16 cell_size;
474 
475 	/* Read Modify Write */
476 	addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
477 
478 	cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size);
479 
480 	ctrl = hinic_hwif_read_reg(hwif, addr);
481 
482 	ctrl =  HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
483 		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR)         &
484 		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN)         &
485 		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN)      &
486 		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
487 
488 	ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR)              |
489 		HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) |
490 		HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE);
491 
492 	hinic_hwif_write_reg(hwif, addr, ctrl);
493 }
494 
495 /**
496  * api_cmd_set_status_addr - set the status address of a chain in the HW
497  * @chain: the API CMD specific chain to set in HW status address for
498  **/
499 static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain)
500 {
501 	struct hinic_hwif *hwif = chain->hwif;
502 	u32 addr, val;
503 
504 	addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type);
505 	val = upper_32_bits(chain->wb_status_paddr);
506 	hinic_hwif_write_reg(hwif, addr, val);
507 
508 	addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type);
509 	val = lower_32_bits(chain->wb_status_paddr);
510 	hinic_hwif_write_reg(hwif, addr, val);
511 }
512 
513 /**
514  * api_cmd_set_num_cells - set the number cells of a chain in the HW
515  * @chain: the API CMD specific chain to set in HW the number of cells for
516  **/
517 static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain)
518 {
519 	struct hinic_hwif *hwif = chain->hwif;
520 	u32 addr, val;
521 
522 	addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type);
523 	val  = chain->num_cells;
524 	hinic_hwif_write_reg(hwif, addr, val);
525 }
526 
527 /**
528  * api_cmd_head_init - set the head of a chain in the HW
529  * @chain: the API CMD specific chain to set in HW the head for
530  **/
531 static void api_cmd_head_init(struct hinic_api_cmd_chain *chain)
532 {
533 	struct hinic_hwif *hwif = chain->hwif;
534 	u32 addr, val;
535 
536 	addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type);
537 	val = upper_32_bits(chain->head_cell_paddr);
538 	hinic_hwif_write_reg(hwif, addr, val);
539 
540 	addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type);
541 	val = lower_32_bits(chain->head_cell_paddr);
542 	hinic_hwif_write_reg(hwif, addr, val);
543 }
544 
545 /**
546  * api_cmd_chain_hw_clean - clean the HW
547  * @chain: the API CMD specific chain
548  **/
549 static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain)
550 {
551 	struct hinic_hwif *hwif = chain->hwif;
552 	u32 addr, ctrl;
553 
554 	addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
555 
556 	ctrl = hinic_hwif_read_reg(hwif, addr);
557 	ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
558 	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR)         &
559 	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN)         &
560 	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN)      &
561 	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
562 
563 	hinic_hwif_write_reg(hwif, addr, ctrl);
564 }
565 
566 /**
567  * api_cmd_chain_hw_init - initialize the chain in the HW
568  * @chain: the API CMD specific chain to initialize in HW
569  *
570  * Return 0 - Success, negative - Failure
571  **/
572 static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain)
573 {
574 	struct hinic_hwif *hwif = chain->hwif;
575 	struct pci_dev *pdev = hwif->pdev;
576 	int err;
577 
578 	api_cmd_chain_hw_clean(chain);
579 
580 	api_cmd_set_status_addr(chain);
581 
582 	err = api_cmd_hw_restart(chain);
583 	if (err) {
584 		dev_err(&pdev->dev, "Failed to restart API CMD HW\n");
585 		return err;
586 	}
587 
588 	api_cmd_ctrl_init(chain);
589 	api_cmd_set_num_cells(chain);
590 	api_cmd_head_init(chain);
591 	return 0;
592 }
593 
594 /**
595  * free_cmd_buf - free the dma buffer of API CMD command
596  * @chain: the API CMD specific chain of the cmd
597  * @cell_idx: the cell index of the cmd
598  **/
599 static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx)
600 {
601 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
602 	struct hinic_hwif *hwif = chain->hwif;
603 	struct pci_dev *pdev = hwif->pdev;
604 
605 	cell_ctxt = &chain->cell_ctxt[cell_idx];
606 
607 	dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE,
608 			  cell_ctxt->api_cmd_vaddr,
609 			  cell_ctxt->api_cmd_paddr);
610 }
611 
612 /**
613  * alloc_cmd_buf - allocate a dma buffer for API CMD command
614  * @chain: the API CMD specific chain for the cmd
615  * @cell: the cell in the HW for the cmd
616  * @cell_idx: the index of the cell
617  *
618  * Return 0 - Success, negative - Failure
619  **/
620 static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
621 			 struct hinic_api_cmd_cell *cell, int cell_idx)
622 {
623 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
624 	struct hinic_hwif *hwif = chain->hwif;
625 	struct pci_dev *pdev = hwif->pdev;
626 	dma_addr_t cmd_paddr;
627 	u8 *cmd_vaddr;
628 	int err = 0;
629 
630 	cmd_vaddr = dma_alloc_coherent(&pdev->dev, API_CMD_BUF_SIZE,
631 				       &cmd_paddr, GFP_KERNEL);
632 	if (!cmd_vaddr)
633 		return -ENOMEM;
634 
635 	cell_ctxt = &chain->cell_ctxt[cell_idx];
636 
637 	cell_ctxt->api_cmd_vaddr = cmd_vaddr;
638 	cell_ctxt->api_cmd_paddr = cmd_paddr;
639 
640 	/* set the cmd DMA address in the cell */
641 	switch (chain->chain_type) {
642 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
643 		/* The data in the HW should be in Big Endian Format */
644 		cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr);
645 		break;
646 
647 	default:
648 		dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
649 		free_cmd_buf(chain, cell_idx);
650 		err = -EINVAL;
651 		break;
652 	}
653 
654 	return err;
655 }
656 
657 /**
658  * api_cmd_create_cell - create API CMD cell for specific chain
659  * @chain: the API CMD specific chain to create its cell
660  * @cell_idx: the index of the cell to create
661  * @pre_node: previous cell
662  * @node_vaddr: the returned virt addr of the cell
663  *
664  * Return 0 - Success, negative - Failure
665  **/
666 static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
667 			       int cell_idx,
668 			       struct hinic_api_cmd_cell *pre_node,
669 			       struct hinic_api_cmd_cell **node_vaddr)
670 {
671 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
672 	struct hinic_hwif *hwif = chain->hwif;
673 	struct pci_dev *pdev = hwif->pdev;
674 	struct hinic_api_cmd_cell *node;
675 	dma_addr_t node_paddr;
676 	int err;
677 
678 	node = dma_alloc_coherent(&pdev->dev, chain->cell_size, &node_paddr,
679 				  GFP_KERNEL);
680 	if (!node)
681 		return -ENOMEM;
682 
683 	node->read.hw_wb_resp_paddr = 0;
684 
685 	cell_ctxt = &chain->cell_ctxt[cell_idx];
686 	cell_ctxt->cell_vaddr = node;
687 	cell_ctxt->cell_paddr = node_paddr;
688 
689 	if (!pre_node) {
690 		chain->head_cell_paddr = node_paddr;
691 		chain->head_node = node;
692 	} else {
693 		/* The data in the HW should be in Big Endian Format */
694 		pre_node->next_cell_paddr = cpu_to_be64(node_paddr);
695 	}
696 
697 	switch (chain->chain_type) {
698 	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
699 		err = alloc_cmd_buf(chain, node, cell_idx);
700 		if (err) {
701 			dev_err(&pdev->dev, "Failed to allocate cmd buffer\n");
702 			goto err_alloc_cmd_buf;
703 		}
704 		break;
705 
706 	default:
707 		dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
708 		err = -EINVAL;
709 		goto err_alloc_cmd_buf;
710 	}
711 
712 	*node_vaddr = node;
713 	return 0;
714 
715 err_alloc_cmd_buf:
716 	dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr);
717 	return err;
718 }
719 
720 /**
721  * api_cmd_destroy_cell - destroy API CMD cell of specific chain
722  * @chain: the API CMD specific chain to destroy its cell
723  * @cell_idx: the cell to destroy
724  **/
725 static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain,
726 				 int cell_idx)
727 {
728 	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
729 	struct hinic_hwif *hwif = chain->hwif;
730 	struct pci_dev *pdev = hwif->pdev;
731 	struct hinic_api_cmd_cell *node;
732 	dma_addr_t node_paddr;
733 	size_t node_size;
734 
735 	cell_ctxt = &chain->cell_ctxt[cell_idx];
736 
737 	node = cell_ctxt->cell_vaddr;
738 	node_paddr = cell_ctxt->cell_paddr;
739 	node_size = chain->cell_size;
740 
741 	if (cell_ctxt->api_cmd_vaddr) {
742 		switch (chain->chain_type) {
743 		case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
744 			free_cmd_buf(chain, cell_idx);
745 			break;
746 		default:
747 			dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
748 			break;
749 		}
750 
751 		dma_free_coherent(&pdev->dev, node_size, node,
752 				  node_paddr);
753 	}
754 }
755 
756 /**
757  * api_cmd_destroy_cells - destroy API CMD cells of specific chain
758  * @chain: the API CMD specific chain to destroy its cells
759  * @num_cells: number of cells to destroy
760  **/
761 static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain,
762 				  int num_cells)
763 {
764 	int cell_idx;
765 
766 	for (cell_idx = 0; cell_idx < num_cells; cell_idx++)
767 		api_cmd_destroy_cell(chain, cell_idx);
768 }
769 
770 /**
771  * api_cmd_create_cells - create API CMD cells for specific chain
772  * @chain: the API CMD specific chain
773  *
774  * Return 0 - Success, negative - Failure
775  **/
776 static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain)
777 {
778 	struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL;
779 	struct hinic_hwif *hwif = chain->hwif;
780 	struct pci_dev *pdev = hwif->pdev;
781 	int err, cell_idx;
782 
783 	for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) {
784 		err = api_cmd_create_cell(chain, cell_idx, pre_node, &node);
785 		if (err) {
786 			dev_err(&pdev->dev, "Failed to create API CMD cell\n");
787 			goto err_create_cell;
788 		}
789 
790 		pre_node = node;
791 	}
792 
793 	/* set the Final node to point on the start */
794 	node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr);
795 
796 	/* set the current node to be the head */
797 	chain->curr_node = chain->head_node;
798 	return 0;
799 
800 err_create_cell:
801 	api_cmd_destroy_cells(chain, cell_idx);
802 	return err;
803 }
804 
805 /**
806  * api_chain_init - initialize API CMD specific chain
807  * @chain: the API CMD specific chain to initialize
808  * @attr: attributes to set in the chain
809  *
810  * Return 0 - Success, negative - Failure
811  **/
812 static int api_chain_init(struct hinic_api_cmd_chain *chain,
813 			  struct hinic_api_cmd_chain_attr *attr)
814 {
815 	struct hinic_hwif *hwif = attr->hwif;
816 	struct pci_dev *pdev = hwif->pdev;
817 	size_t cell_ctxt_size;
818 
819 	chain->hwif = hwif;
820 	chain->chain_type  = attr->chain_type;
821 	chain->num_cells = attr->num_cells;
822 	chain->cell_size = attr->cell_size;
823 
824 	chain->prod_idx  = 0;
825 	chain->cons_idx  = 0;
826 
827 	sema_init(&chain->sem, 1);
828 
829 	cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt);
830 	chain->cell_ctxt = devm_kzalloc(&pdev->dev, cell_ctxt_size, GFP_KERNEL);
831 	if (!chain->cell_ctxt)
832 		return -ENOMEM;
833 
834 	chain->wb_status = dma_alloc_coherent(&pdev->dev,
835 					      sizeof(*chain->wb_status),
836 					      &chain->wb_status_paddr,
837 					      GFP_KERNEL);
838 	if (!chain->wb_status) {
839 		dev_err(&pdev->dev, "Failed to allocate DMA wb status\n");
840 		return -ENOMEM;
841 	}
842 
843 	return 0;
844 }
845 
846 /**
847  * api_chain_free - free API CMD specific chain
848  * @chain: the API CMD specific chain to free
849  **/
850 static void api_chain_free(struct hinic_api_cmd_chain *chain)
851 {
852 	struct hinic_hwif *hwif = chain->hwif;
853 	struct pci_dev *pdev = hwif->pdev;
854 
855 	dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status),
856 			  chain->wb_status, chain->wb_status_paddr);
857 }
858 
859 /**
860  * api_cmd_create_chain - create API CMD specific chain
861  * @attr: attributes to set the chain
862  *
863  * Return the created chain
864  **/
865 static struct hinic_api_cmd_chain *
866 	api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr)
867 {
868 	struct hinic_hwif *hwif = attr->hwif;
869 	struct pci_dev *pdev = hwif->pdev;
870 	struct hinic_api_cmd_chain *chain;
871 	int err;
872 
873 	if (attr->num_cells & (attr->num_cells - 1)) {
874 		dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n");
875 		return ERR_PTR(-EINVAL);
876 	}
877 
878 	chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
879 	if (!chain)
880 		return ERR_PTR(-ENOMEM);
881 
882 	err = api_chain_init(chain, attr);
883 	if (err) {
884 		dev_err(&pdev->dev, "Failed to initialize chain\n");
885 		return ERR_PTR(err);
886 	}
887 
888 	err = api_cmd_create_cells(chain);
889 	if (err) {
890 		dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n");
891 		goto err_create_cells;
892 	}
893 
894 	err = api_cmd_chain_hw_init(chain);
895 	if (err) {
896 		dev_err(&pdev->dev, "Failed to initialize chain HW\n");
897 		goto err_chain_hw_init;
898 	}
899 
900 	return chain;
901 
902 err_chain_hw_init:
903 	api_cmd_destroy_cells(chain, chain->num_cells);
904 
905 err_create_cells:
906 	api_chain_free(chain);
907 	return ERR_PTR(err);
908 }
909 
910 /**
911  * api_cmd_destroy_chain - destroy API CMD specific chain
912  * @chain: the API CMD specific chain to destroy
913  **/
914 static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain)
915 {
916 	api_cmd_chain_hw_clean(chain);
917 	api_cmd_destroy_cells(chain, chain->num_cells);
918 	api_chain_free(chain);
919 }
920 
921 /**
922  * hinic_api_cmd_init - Initialize all the API CMD chains
923  * @chain: the API CMD chains that are initialized
924  * @hwif: the hardware interface of a pci function device
925  *
926  * Return 0 - Success, negative - Failure
927  **/
928 int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain,
929 		       struct hinic_hwif *hwif)
930 {
931 	enum hinic_api_cmd_chain_type type, chain_type;
932 	struct hinic_api_cmd_chain_attr attr;
933 	struct pci_dev *pdev = hwif->pdev;
934 	size_t hw_cell_sz;
935 	int err;
936 
937 	hw_cell_sz = sizeof(struct hinic_api_cmd_cell);
938 
939 	attr.hwif = hwif;
940 	attr.num_cells  = API_CHAIN_NUM_CELLS;
941 	attr.cell_size  = API_CMD_CELL_SIZE(hw_cell_sz);
942 
943 	chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
944 	for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
945 		attr.chain_type = chain_type;
946 
947 		if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
948 			continue;
949 
950 		chain[chain_type] = api_cmd_create_chain(&attr);
951 		if (IS_ERR(chain[chain_type])) {
952 			dev_err(&pdev->dev, "Failed to create chain %d\n",
953 				chain_type);
954 			err = PTR_ERR(chain[chain_type]);
955 			goto err_create_chain;
956 		}
957 	}
958 
959 	return 0;
960 
961 err_create_chain:
962 	type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
963 	for ( ; type < chain_type; type++) {
964 		if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
965 			continue;
966 
967 		api_cmd_destroy_chain(chain[type]);
968 	}
969 
970 	return err;
971 }
972 
973 /**
974  * hinic_api_cmd_free - free the API CMD chains
975  * @chain: the API CMD chains that are freed
976  **/
977 void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain)
978 {
979 	enum hinic_api_cmd_chain_type chain_type;
980 
981 	chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
982 	for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
983 		if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
984 			continue;
985 
986 		api_cmd_destroy_chain(chain[chain_type]);
987 	}
988 }
989