1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Huawei HiNIC PCI Express Linux driver
4  * Copyright(c) 2017 Huawei Technologies Co., Ltd
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/errno.h>
10 #include <linux/pci.h>
11 #include <linux/device.h>
12 #include <linux/workqueue.h>
13 #include <linux/interrupt.h>
14 #include <linux/slab.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/log2.h>
17 #include <asm/byteorder.h>
18 #include <asm/barrier.h>
19 
20 #include "hinic_hw_dev.h"
21 #include "hinic_hw_csr.h"
22 #include "hinic_hw_if.h"
23 #include "hinic_hw_eqs.h"
24 
25 #define HINIC_EQS_WQ_NAME                       "hinic_eqs"
26 
27 #define GET_EQ_NUM_PAGES(eq, pg_size)           \
28 		(ALIGN((eq)->q_len * (eq)->elem_size, pg_size) / (pg_size))
29 
30 #define GET_EQ_NUM_ELEMS_IN_PG(eq, pg_size)     ((pg_size) / (eq)->elem_size)
31 
32 #define EQ_CONS_IDX_REG_ADDR(eq)        (((eq)->type == HINIC_AEQ) ? \
33 			HINIC_CSR_AEQ_CONS_IDX_ADDR((eq)->q_id) : \
34 			HINIC_CSR_CEQ_CONS_IDX_ADDR((eq)->q_id))
35 
36 #define EQ_PROD_IDX_REG_ADDR(eq)        (((eq)->type == HINIC_AEQ) ? \
37 			HINIC_CSR_AEQ_PROD_IDX_ADDR((eq)->q_id) : \
38 			HINIC_CSR_CEQ_PROD_IDX_ADDR((eq)->q_id))
39 
40 #define EQ_HI_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \
41 			HINIC_CSR_AEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num) : \
42 			HINIC_CSR_CEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num))
43 
44 #define EQ_LO_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \
45 			HINIC_CSR_AEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num) : \
46 			HINIC_CSR_CEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num))
47 
48 #define GET_EQ_ELEMENT(eq, idx)         \
49 		((eq)->virt_addr[(idx) / (eq)->num_elem_in_pg] + \
50 		 (((idx) & ((eq)->num_elem_in_pg - 1)) * (eq)->elem_size))
51 
52 #define GET_AEQ_ELEM(eq, idx)           ((struct hinic_aeq_elem *) \
53 					GET_EQ_ELEMENT(eq, idx))
54 
55 #define GET_CEQ_ELEM(eq, idx)           ((u32 *) \
56 					 GET_EQ_ELEMENT(eq, idx))
57 
58 #define GET_CURR_AEQ_ELEM(eq)           GET_AEQ_ELEM(eq, (eq)->cons_idx)
59 
60 #define GET_CURR_CEQ_ELEM(eq)           GET_CEQ_ELEM(eq, (eq)->cons_idx)
61 
62 #define PAGE_IN_4K(page_size)           ((page_size) >> 12)
63 #define EQ_SET_HW_PAGE_SIZE_VAL(eq)     (ilog2(PAGE_IN_4K((eq)->page_size)))
64 
65 #define ELEMENT_SIZE_IN_32B(eq)         (((eq)->elem_size) >> 5)
66 #define EQ_SET_HW_ELEM_SIZE_VAL(eq)     (ilog2(ELEMENT_SIZE_IN_32B(eq)))
67 
68 #define EQ_MAX_PAGES                    8
69 
70 #define CEQE_TYPE_SHIFT                 23
71 #define CEQE_TYPE_MASK                  0x7
72 
73 #define CEQE_TYPE(ceqe)                 (((ceqe) >> CEQE_TYPE_SHIFT) &  \
74 					 CEQE_TYPE_MASK)
75 
76 #define CEQE_DATA_MASK                  0x3FFFFFF
77 #define CEQE_DATA(ceqe)                 ((ceqe) & CEQE_DATA_MASK)
78 
79 #define aeq_to_aeqs(eq)                 \
80 		container_of((eq) - (eq)->q_id, struct hinic_aeqs, aeq[0])
81 
82 #define ceq_to_ceqs(eq)                 \
83 		container_of((eq) - (eq)->q_id, struct hinic_ceqs, ceq[0])
84 
85 #define work_to_aeq_work(work)          \
86 		container_of(work, struct hinic_eq_work, work)
87 
88 #define DMA_ATTR_AEQ_DEFAULT            0
89 #define DMA_ATTR_CEQ_DEFAULT            0
90 
91 /* No coalescence */
92 #define THRESH_CEQ_DEFAULT              0
93 
94 enum eq_int_mode {
95 	EQ_INT_MODE_ARMED,
96 	EQ_INT_MODE_ALWAYS
97 };
98 
99 enum eq_arm_state {
100 	EQ_NOT_ARMED,
101 	EQ_ARMED
102 };
103 
104 /**
105  * hinic_aeq_register_hw_cb - register AEQ callback for specific event
106  * @aeqs: pointer to Async eqs of the chip
107  * @event: aeq event to register callback for it
108  * @handle: private data will be used by the callback
109  * @hwe_handler: callback function
110  **/
111 void hinic_aeq_register_hw_cb(struct hinic_aeqs *aeqs,
112 			      enum hinic_aeq_type event, void *handle,
113 			      void (*hwe_handler)(void *handle, void *data,
114 						  u8 size))
115 {
116 	struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event];
117 
118 	hwe_cb->hwe_handler = hwe_handler;
119 	hwe_cb->handle = handle;
120 	hwe_cb->hwe_state = HINIC_EQE_ENABLED;
121 }
122 
123 /**
124  * hinic_aeq_unregister_hw_cb - unregister the AEQ callback for specific event
125  * @aeqs: pointer to Async eqs of the chip
126  * @event: aeq event to unregister callback for it
127  **/
128 void hinic_aeq_unregister_hw_cb(struct hinic_aeqs *aeqs,
129 				enum hinic_aeq_type event)
130 {
131 	struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event];
132 
133 	hwe_cb->hwe_state &= ~HINIC_EQE_ENABLED;
134 
135 	while (hwe_cb->hwe_state & HINIC_EQE_RUNNING)
136 		schedule();
137 
138 	hwe_cb->hwe_handler = NULL;
139 }
140 
141 /**
142  * hinic_ceq_register_cb - register CEQ callback for specific event
143  * @ceqs: pointer to Completion eqs part of the chip
144  * @event: ceq event to register callback for it
145  * @handle: private data will be used by the callback
146  * @handler: callback function
147  **/
148 void hinic_ceq_register_cb(struct hinic_ceqs *ceqs,
149 			   enum hinic_ceq_type event, void *handle,
150 			   void (*handler)(void *handle, u32 ceqe_data))
151 {
152 	struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event];
153 
154 	ceq_cb->handler = handler;
155 	ceq_cb->handle = handle;
156 	ceq_cb->ceqe_state = HINIC_EQE_ENABLED;
157 }
158 
159 /**
160  * hinic_ceq_unregister_cb - unregister the CEQ callback for specific event
161  * @ceqs: pointer to Completion eqs part of the chip
162  * @event: ceq event to unregister callback for it
163  **/
164 void hinic_ceq_unregister_cb(struct hinic_ceqs *ceqs,
165 			     enum hinic_ceq_type event)
166 {
167 	struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event];
168 
169 	ceq_cb->ceqe_state &= ~HINIC_EQE_ENABLED;
170 
171 	while (ceq_cb->ceqe_state & HINIC_EQE_RUNNING)
172 		schedule();
173 
174 	ceq_cb->handler = NULL;
175 }
176 
177 static u8 eq_cons_idx_checksum_set(u32 val)
178 {
179 	u8 checksum = 0;
180 	int idx;
181 
182 	for (idx = 0; idx < 32; idx += 4)
183 		checksum ^= ((val >> idx) & 0xF);
184 
185 	return (checksum & 0xF);
186 }
187 
188 /**
189  * eq_update_ci - update the HW cons idx of event queue
190  * @eq: the event queue to update the cons idx for
191  * @arm_state: the arm bit value of eq's interrupt
192  **/
193 static void eq_update_ci(struct hinic_eq *eq, u32 arm_state)
194 {
195 	u32 val, addr = EQ_CONS_IDX_REG_ADDR(eq);
196 
197 	/* Read Modify Write */
198 	val = hinic_hwif_read_reg(eq->hwif, addr);
199 
200 	val = HINIC_EQ_CI_CLEAR(val, IDX)       &
201 	      HINIC_EQ_CI_CLEAR(val, WRAPPED)   &
202 	      HINIC_EQ_CI_CLEAR(val, INT_ARMED) &
203 	      HINIC_EQ_CI_CLEAR(val, XOR_CHKSUM);
204 
205 	val |= HINIC_EQ_CI_SET(eq->cons_idx, IDX)    |
206 	       HINIC_EQ_CI_SET(eq->wrapped, WRAPPED) |
207 	       HINIC_EQ_CI_SET(arm_state, INT_ARMED);
208 
209 	val |= HINIC_EQ_CI_SET(eq_cons_idx_checksum_set(val), XOR_CHKSUM);
210 
211 	hinic_hwif_write_reg(eq->hwif, addr, val);
212 }
213 
214 /**
215  * aeq_irq_handler - handler for the AEQ event
216  * @eq: the Async Event Queue that received the event
217  **/
218 static void aeq_irq_handler(struct hinic_eq *eq)
219 {
220 	struct hinic_aeqs *aeqs = aeq_to_aeqs(eq);
221 	struct hinic_hwif *hwif = aeqs->hwif;
222 	struct pci_dev *pdev = hwif->pdev;
223 	struct hinic_aeq_elem *aeqe_curr;
224 	struct hinic_hw_event_cb *hwe_cb;
225 	enum hinic_aeq_type event;
226 	unsigned long eqe_state;
227 	u32 aeqe_desc;
228 	int i, size;
229 
230 	for (i = 0; i < eq->q_len; i++) {
231 		aeqe_curr = GET_CURR_AEQ_ELEM(eq);
232 
233 		/* Data in HW is in Big endian Format */
234 		aeqe_desc = be32_to_cpu(aeqe_curr->desc);
235 
236 		/* HW toggles the wrapped bit, when it adds eq element */
237 		if (HINIC_EQ_ELEM_DESC_GET(aeqe_desc, WRAPPED) == eq->wrapped)
238 			break;
239 
240 		dma_rmb();
241 
242 		event = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, TYPE);
243 		if (event >= HINIC_MAX_AEQ_EVENTS) {
244 			dev_err(&pdev->dev, "Unknown AEQ Event %d\n", event);
245 			return;
246 		}
247 
248 		if (!HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SRC)) {
249 			hwe_cb = &aeqs->hwe_cb[event];
250 
251 			size = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SIZE);
252 
253 			eqe_state = cmpxchg(&hwe_cb->hwe_state,
254 					    HINIC_EQE_ENABLED,
255 					    HINIC_EQE_ENABLED |
256 					    HINIC_EQE_RUNNING);
257 			if (eqe_state == HINIC_EQE_ENABLED &&
258 			    hwe_cb->hwe_handler)
259 				hwe_cb->hwe_handler(hwe_cb->handle,
260 						    aeqe_curr->data, size);
261 			else
262 				dev_err(&pdev->dev, "Unhandled AEQ Event %d\n",
263 					event);
264 
265 			hwe_cb->hwe_state &= ~HINIC_EQE_RUNNING;
266 		}
267 
268 		eq->cons_idx++;
269 
270 		if (eq->cons_idx == eq->q_len) {
271 			eq->cons_idx = 0;
272 			eq->wrapped = !eq->wrapped;
273 		}
274 	}
275 }
276 
277 /**
278  * ceq_event_handler - handler for the ceq events
279  * @ceqs: ceqs part of the chip
280  * @ceqe: ceq element that describes the event
281  **/
282 static void ceq_event_handler(struct hinic_ceqs *ceqs, u32 ceqe)
283 {
284 	struct hinic_hwif *hwif = ceqs->hwif;
285 	struct pci_dev *pdev = hwif->pdev;
286 	struct hinic_ceq_cb *ceq_cb;
287 	enum hinic_ceq_type event;
288 	unsigned long eqe_state;
289 
290 	event = CEQE_TYPE(ceqe);
291 	if (event >= HINIC_MAX_CEQ_EVENTS) {
292 		dev_err(&pdev->dev, "Unknown CEQ event, event = %d\n", event);
293 		return;
294 	}
295 
296 	ceq_cb = &ceqs->ceq_cb[event];
297 
298 	eqe_state = cmpxchg(&ceq_cb->ceqe_state,
299 			    HINIC_EQE_ENABLED,
300 			    HINIC_EQE_ENABLED | HINIC_EQE_RUNNING);
301 
302 	if (eqe_state == HINIC_EQE_ENABLED && ceq_cb->handler)
303 		ceq_cb->handler(ceq_cb->handle, CEQE_DATA(ceqe));
304 	else
305 		dev_err(&pdev->dev, "Unhandled CEQ Event %d\n", event);
306 
307 	ceq_cb->ceqe_state &= ~HINIC_EQE_RUNNING;
308 }
309 
310 /**
311  * ceq_irq_handler - handler for the CEQ event
312  * @eq: the Completion Event Queue that received the event
313  **/
314 static void ceq_irq_handler(struct hinic_eq *eq)
315 {
316 	struct hinic_ceqs *ceqs = ceq_to_ceqs(eq);
317 	u32 ceqe;
318 	int i;
319 
320 	for (i = 0; i < eq->q_len; i++) {
321 		ceqe = *(GET_CURR_CEQ_ELEM(eq));
322 
323 		/* Data in HW is in Big endian Format */
324 		ceqe = be32_to_cpu(ceqe);
325 
326 		/* HW toggles the wrapped bit, when it adds eq element event */
327 		if (HINIC_EQ_ELEM_DESC_GET(ceqe, WRAPPED) == eq->wrapped)
328 			break;
329 
330 		ceq_event_handler(ceqs, ceqe);
331 
332 		eq->cons_idx++;
333 
334 		if (eq->cons_idx == eq->q_len) {
335 			eq->cons_idx = 0;
336 			eq->wrapped = !eq->wrapped;
337 		}
338 	}
339 }
340 
341 /**
342  * eq_irq_handler - handler for the EQ event
343  * @data: the Event Queue that received the event
344  **/
345 static void eq_irq_handler(void *data)
346 {
347 	struct hinic_eq *eq = data;
348 
349 	if (eq->type == HINIC_AEQ)
350 		aeq_irq_handler(eq);
351 	else if (eq->type == HINIC_CEQ)
352 		ceq_irq_handler(eq);
353 
354 	eq_update_ci(eq, EQ_ARMED);
355 }
356 
357 /**
358  * eq_irq_work - the work of the EQ that received the event
359  * @work: the work struct that is associated with the EQ
360  **/
361 static void eq_irq_work(struct work_struct *work)
362 {
363 	struct hinic_eq_work *aeq_work = work_to_aeq_work(work);
364 	struct hinic_eq *aeq;
365 
366 	aeq = aeq_work->data;
367 	eq_irq_handler(aeq);
368 }
369 
370 /**
371  * ceq_tasklet - the tasklet of the EQ that received the event
372  * @t: the tasklet struct pointer
373  **/
374 static void ceq_tasklet(struct tasklet_struct *t)
375 {
376 	struct hinic_eq *ceq = from_tasklet(ceq, t, ceq_tasklet);
377 
378 	eq_irq_handler(ceq);
379 }
380 
381 /**
382  * aeq_interrupt - aeq interrupt handler
383  * @irq: irq number
384  * @data: the Async Event Queue that collected the event
385  **/
386 static irqreturn_t aeq_interrupt(int irq, void *data)
387 {
388 	struct hinic_eq_work *aeq_work;
389 	struct hinic_eq *aeq = data;
390 	struct hinic_aeqs *aeqs;
391 
392 	/* clear resend timer cnt register */
393 	hinic_msix_attr_cnt_clear(aeq->hwif, aeq->msix_entry.entry);
394 
395 	aeq_work = &aeq->aeq_work;
396 	aeq_work->data = aeq;
397 
398 	aeqs = aeq_to_aeqs(aeq);
399 	queue_work(aeqs->workq, &aeq_work->work);
400 
401 	return IRQ_HANDLED;
402 }
403 
404 /**
405  * ceq_interrupt - ceq interrupt handler
406  * @irq: irq number
407  * @data: the Completion Event Queue that collected the event
408  **/
409 static irqreturn_t ceq_interrupt(int irq, void *data)
410 {
411 	struct hinic_eq *ceq = data;
412 
413 	/* clear resend timer cnt register */
414 	hinic_msix_attr_cnt_clear(ceq->hwif, ceq->msix_entry.entry);
415 
416 	tasklet_schedule(&ceq->ceq_tasklet);
417 
418 	return IRQ_HANDLED;
419 }
420 
421 static u32 get_ctrl0_val(struct hinic_eq *eq, u32 addr)
422 {
423 	struct msix_entry *msix_entry = &eq->msix_entry;
424 	enum hinic_eq_type type = eq->type;
425 	u32 val, ctrl0;
426 
427 	if (type == HINIC_AEQ) {
428 		/* RMW Ctrl0 */
429 		addr = HINIC_CSR_AEQ_CTRL_0_ADDR(eq->q_id);
430 
431 		val = hinic_hwif_read_reg(eq->hwif, addr);
432 
433 		val = HINIC_AEQ_CTRL_0_CLEAR(val, INT_IDX)      &
434 		      HINIC_AEQ_CTRL_0_CLEAR(val, DMA_ATTR)     &
435 		      HINIC_AEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) &
436 		      HINIC_AEQ_CTRL_0_CLEAR(val, INT_MODE);
437 
438 		ctrl0 = HINIC_AEQ_CTRL_0_SET(msix_entry->entry, INT_IDX)     |
439 			HINIC_AEQ_CTRL_0_SET(DMA_ATTR_AEQ_DEFAULT, DMA_ATTR) |
440 			HINIC_AEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif),
441 					     PCI_INTF_IDX)                   |
442 			HINIC_AEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INT_MODE);
443 
444 		val |= ctrl0;
445 	} else {
446 		/* RMW Ctrl0 */
447 		addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id);
448 
449 		val = hinic_hwif_read_reg(eq->hwif, addr);
450 
451 		val = HINIC_CEQ_CTRL_0_CLEAR(val, INTR_IDX)     &
452 		      HINIC_CEQ_CTRL_0_CLEAR(val, DMA_ATTR)     &
453 		      HINIC_CEQ_CTRL_0_CLEAR(val, KICK_THRESH)  &
454 		      HINIC_CEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) &
455 		      HINIC_CEQ_CTRL_0_CLEAR(val, INTR_MODE);
456 
457 		ctrl0 = HINIC_CEQ_CTRL_0_SET(msix_entry->entry, INTR_IDX)     |
458 			HINIC_CEQ_CTRL_0_SET(DMA_ATTR_CEQ_DEFAULT, DMA_ATTR)  |
459 			HINIC_CEQ_CTRL_0_SET(THRESH_CEQ_DEFAULT, KICK_THRESH) |
460 			HINIC_CEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif),
461 					     PCI_INTF_IDX)                    |
462 			HINIC_CEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INTR_MODE);
463 
464 		val |= ctrl0;
465 	}
466 	return val;
467 }
468 
469 static void set_ctrl0(struct hinic_eq *eq)
470 {
471 	u32 val, addr;
472 
473 	if (eq->type == HINIC_AEQ)
474 		addr = HINIC_CSR_AEQ_CTRL_0_ADDR(eq->q_id);
475 	else
476 		addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id);
477 
478 	val = get_ctrl0_val(eq, addr);
479 
480 	hinic_hwif_write_reg(eq->hwif, addr, val);
481 }
482 
483 static u32 get_ctrl1_val(struct hinic_eq *eq, u32 addr)
484 {
485 	u32 page_size_val, elem_size, val, ctrl1;
486 	enum hinic_eq_type type = eq->type;
487 
488 	if (type == HINIC_AEQ) {
489 		/* RMW Ctrl1 */
490 		addr = HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id);
491 
492 		page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq);
493 		elem_size = EQ_SET_HW_ELEM_SIZE_VAL(eq);
494 
495 		val = hinic_hwif_read_reg(eq->hwif, addr);
496 
497 		val = HINIC_AEQ_CTRL_1_CLEAR(val, LEN)          &
498 		      HINIC_AEQ_CTRL_1_CLEAR(val, ELEM_SIZE)    &
499 		      HINIC_AEQ_CTRL_1_CLEAR(val, PAGE_SIZE);
500 
501 		ctrl1 = HINIC_AEQ_CTRL_1_SET(eq->q_len, LEN)            |
502 			HINIC_AEQ_CTRL_1_SET(elem_size, ELEM_SIZE)      |
503 			HINIC_AEQ_CTRL_1_SET(page_size_val, PAGE_SIZE);
504 
505 		val |= ctrl1;
506 	} else {
507 		/* RMW Ctrl1 */
508 		addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id);
509 
510 		page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq);
511 
512 		val = hinic_hwif_read_reg(eq->hwif, addr);
513 
514 		val = HINIC_CEQ_CTRL_1_CLEAR(val, LEN) &
515 		      HINIC_CEQ_CTRL_1_CLEAR(val, PAGE_SIZE);
516 
517 		ctrl1 = HINIC_CEQ_CTRL_1_SET(eq->q_len, LEN) |
518 			HINIC_CEQ_CTRL_1_SET(page_size_val, PAGE_SIZE);
519 
520 		val |= ctrl1;
521 	}
522 	return val;
523 }
524 
525 static void set_ctrl1(struct hinic_eq *eq)
526 {
527 	u32 addr, val;
528 
529 	if (eq->type == HINIC_AEQ)
530 		addr = HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id);
531 	else
532 		addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id);
533 
534 	val = get_ctrl1_val(eq, addr);
535 
536 	hinic_hwif_write_reg(eq->hwif, addr, val);
537 }
538 
539 static int set_ceq_ctrl_reg(struct hinic_eq *eq)
540 {
541 	struct hinic_ceq_ctrl_reg ceq_ctrl = {0};
542 	struct hinic_hwdev *hwdev = eq->hwdev;
543 	u16 out_size = sizeof(ceq_ctrl);
544 	u16 in_size = sizeof(ceq_ctrl);
545 	struct hinic_pfhwdev *pfhwdev;
546 	u32 addr;
547 	int err;
548 
549 	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
550 
551 	addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id);
552 	ceq_ctrl.ctrl0 = get_ctrl0_val(eq, addr);
553 	addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id);
554 	ceq_ctrl.ctrl1 = get_ctrl1_val(eq, addr);
555 
556 	ceq_ctrl.func_id = HINIC_HWIF_FUNC_IDX(hwdev->hwif);
557 	ceq_ctrl.q_id = eq->q_id;
558 
559 	err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
560 				HINIC_COMM_CMD_CEQ_CTRL_REG_WR_BY_UP,
561 				&ceq_ctrl, in_size,
562 				&ceq_ctrl, &out_size, HINIC_MGMT_MSG_SYNC);
563 	if (err || !out_size || ceq_ctrl.status) {
564 		dev_err(&hwdev->hwif->pdev->dev,
565 			"Failed to set ceq %d ctrl reg, err: %d status: 0x%x, out_size: 0x%x\n",
566 			eq->q_id, err, ceq_ctrl.status, out_size);
567 		return -EFAULT;
568 	}
569 
570 	return 0;
571 }
572 
573 /**
574  * set_eq_ctrls - setting eq's ctrl registers
575  * @eq: the Event Queue for setting
576  **/
577 static int set_eq_ctrls(struct hinic_eq *eq)
578 {
579 	if (HINIC_IS_VF(eq->hwif) && eq->type == HINIC_CEQ)
580 		return set_ceq_ctrl_reg(eq);
581 
582 	set_ctrl0(eq);
583 	set_ctrl1(eq);
584 	return 0;
585 }
586 
587 /**
588  * aeq_elements_init - initialize all the elements in the aeq
589  * @eq: the Async Event Queue
590  * @init_val: value to initialize the elements with it
591  **/
592 static void aeq_elements_init(struct hinic_eq *eq, u32 init_val)
593 {
594 	struct hinic_aeq_elem *aeqe;
595 	int i;
596 
597 	for (i = 0; i < eq->q_len; i++) {
598 		aeqe = GET_AEQ_ELEM(eq, i);
599 		aeqe->desc = cpu_to_be32(init_val);
600 	}
601 
602 	wmb();  /* Write the initilzation values */
603 }
604 
605 /**
606  * ceq_elements_init - Initialize all the elements in the ceq
607  * @eq: the event queue
608  * @init_val: value to init with it the elements
609  **/
610 static void ceq_elements_init(struct hinic_eq *eq, u32 init_val)
611 {
612 	u32 *ceqe;
613 	int i;
614 
615 	for (i = 0; i < eq->q_len; i++) {
616 		ceqe = GET_CEQ_ELEM(eq, i);
617 		*(ceqe) = cpu_to_be32(init_val);
618 	}
619 
620 	wmb();  /* Write the initilzation values */
621 }
622 
623 /**
624  * alloc_eq_pages - allocate the pages for the queue
625  * @eq: the event queue
626  *
627  * Return 0 - Success, Negative - Failure
628  **/
629 static int alloc_eq_pages(struct hinic_eq *eq)
630 {
631 	struct hinic_hwif *hwif = eq->hwif;
632 	struct pci_dev *pdev = hwif->pdev;
633 	u32 init_val, addr, val;
634 	size_t addr_size;
635 	int err, pg;
636 
637 	addr_size = eq->num_pages * sizeof(*eq->dma_addr);
638 	eq->dma_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL);
639 	if (!eq->dma_addr)
640 		return -ENOMEM;
641 
642 	addr_size = eq->num_pages * sizeof(*eq->virt_addr);
643 	eq->virt_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL);
644 	if (!eq->virt_addr) {
645 		err = -ENOMEM;
646 		goto err_virt_addr_alloc;
647 	}
648 
649 	for (pg = 0; pg < eq->num_pages; pg++) {
650 		eq->virt_addr[pg] = dma_alloc_coherent(&pdev->dev,
651 						       eq->page_size,
652 						       &eq->dma_addr[pg],
653 						       GFP_KERNEL);
654 		if (!eq->virt_addr[pg]) {
655 			err = -ENOMEM;
656 			goto err_dma_alloc;
657 		}
658 
659 		addr = EQ_HI_PHYS_ADDR_REG(eq, pg);
660 		val = upper_32_bits(eq->dma_addr[pg]);
661 
662 		hinic_hwif_write_reg(hwif, addr, val);
663 
664 		addr = EQ_LO_PHYS_ADDR_REG(eq, pg);
665 		val = lower_32_bits(eq->dma_addr[pg]);
666 
667 		hinic_hwif_write_reg(hwif, addr, val);
668 	}
669 
670 	init_val = HINIC_EQ_ELEM_DESC_SET(eq->wrapped, WRAPPED);
671 
672 	if (eq->type == HINIC_AEQ)
673 		aeq_elements_init(eq, init_val);
674 	else if (eq->type == HINIC_CEQ)
675 		ceq_elements_init(eq, init_val);
676 
677 	return 0;
678 
679 err_dma_alloc:
680 	while (--pg >= 0)
681 		dma_free_coherent(&pdev->dev, eq->page_size,
682 				  eq->virt_addr[pg],
683 				  eq->dma_addr[pg]);
684 
685 	devm_kfree(&pdev->dev, eq->virt_addr);
686 
687 err_virt_addr_alloc:
688 	devm_kfree(&pdev->dev, eq->dma_addr);
689 	return err;
690 }
691 
692 /**
693  * free_eq_pages - free the pages of the queue
694  * @eq: the Event Queue
695  **/
696 static void free_eq_pages(struct hinic_eq *eq)
697 {
698 	struct hinic_hwif *hwif = eq->hwif;
699 	struct pci_dev *pdev = hwif->pdev;
700 	int pg;
701 
702 	for (pg = 0; pg < eq->num_pages; pg++)
703 		dma_free_coherent(&pdev->dev, eq->page_size,
704 				  eq->virt_addr[pg],
705 				  eq->dma_addr[pg]);
706 
707 	devm_kfree(&pdev->dev, eq->virt_addr);
708 	devm_kfree(&pdev->dev, eq->dma_addr);
709 }
710 
711 /**
712  * init_eq - initialize Event Queue
713  * @eq: the event queue
714  * @hwif: the HW interface of a PCI function device
715  * @type: the type of the event queue, aeq or ceq
716  * @q_id: Queue id number
717  * @q_len: the number of EQ elements
718  * @page_size: the page size of the pages in the event queue
719  * @entry: msix entry associated with the event queue
720  *
721  * Return 0 - Success, Negative - Failure
722  **/
723 static int init_eq(struct hinic_eq *eq, struct hinic_hwif *hwif,
724 		   enum hinic_eq_type type, int q_id, u32 q_len, u32 page_size,
725 		   struct msix_entry entry)
726 {
727 	struct pci_dev *pdev = hwif->pdev;
728 	int err;
729 
730 	eq->hwif = hwif;
731 	eq->type = type;
732 	eq->q_id = q_id;
733 	eq->q_len = q_len;
734 	eq->page_size = page_size;
735 
736 	/* Clear PI and CI, also clear the ARM bit */
737 	hinic_hwif_write_reg(eq->hwif, EQ_CONS_IDX_REG_ADDR(eq), 0);
738 	hinic_hwif_write_reg(eq->hwif, EQ_PROD_IDX_REG_ADDR(eq), 0);
739 
740 	eq->cons_idx = 0;
741 	eq->wrapped = 0;
742 
743 	if (type == HINIC_AEQ) {
744 		eq->elem_size = HINIC_AEQE_SIZE;
745 	} else if (type == HINIC_CEQ) {
746 		eq->elem_size = HINIC_CEQE_SIZE;
747 	} else {
748 		dev_err(&pdev->dev, "Invalid EQ type\n");
749 		return -EINVAL;
750 	}
751 
752 	eq->num_pages = GET_EQ_NUM_PAGES(eq, page_size);
753 	eq->num_elem_in_pg = GET_EQ_NUM_ELEMS_IN_PG(eq, page_size);
754 
755 	eq->msix_entry = entry;
756 
757 	if (eq->num_elem_in_pg & (eq->num_elem_in_pg - 1)) {
758 		dev_err(&pdev->dev, "num elements in eq page != power of 2\n");
759 		return -EINVAL;
760 	}
761 
762 	if (eq->num_pages > EQ_MAX_PAGES) {
763 		dev_err(&pdev->dev, "too many pages for eq\n");
764 		return -EINVAL;
765 	}
766 
767 	err = set_eq_ctrls(eq);
768 	if (err) {
769 		dev_err(&pdev->dev, "Failed to set eq ctrls\n");
770 		return err;
771 	}
772 
773 	eq_update_ci(eq, EQ_ARMED);
774 
775 	err = alloc_eq_pages(eq);
776 	if (err) {
777 		dev_err(&pdev->dev, "Failed to allocate pages for eq\n");
778 		return err;
779 	}
780 
781 	if (type == HINIC_AEQ) {
782 		struct hinic_eq_work *aeq_work = &eq->aeq_work;
783 
784 		INIT_WORK(&aeq_work->work, eq_irq_work);
785 	} else if (type == HINIC_CEQ) {
786 		tasklet_setup(&eq->ceq_tasklet, ceq_tasklet);
787 	}
788 
789 	/* set the attributes of the msix entry */
790 	hinic_msix_attr_set(eq->hwif, eq->msix_entry.entry,
791 			    HINIC_EQ_MSIX_PENDING_LIMIT_DEFAULT,
792 			    HINIC_EQ_MSIX_COALESC_TIMER_DEFAULT,
793 			    HINIC_EQ_MSIX_LLI_TIMER_DEFAULT,
794 			    HINIC_EQ_MSIX_LLI_CREDIT_LIMIT_DEFAULT,
795 			    HINIC_EQ_MSIX_RESEND_TIMER_DEFAULT);
796 
797 	if (type == HINIC_AEQ) {
798 		snprintf(eq->irq_name, sizeof(eq->irq_name), "hinic_aeq%d@pci:%s", eq->q_id,
799 			 pci_name(pdev));
800 		err = request_irq(entry.vector, aeq_interrupt, 0, eq->irq_name, eq);
801 	} else if (type == HINIC_CEQ) {
802 		snprintf(eq->irq_name, sizeof(eq->irq_name), "hinic_ceq%d@pci:%s", eq->q_id,
803 			 pci_name(pdev));
804 		err = request_irq(entry.vector, ceq_interrupt, 0, eq->irq_name, eq);
805 	}
806 
807 	if (err) {
808 		dev_err(&pdev->dev, "Failed to request irq for the EQ\n");
809 		goto err_req_irq;
810 	}
811 
812 	return 0;
813 
814 err_req_irq:
815 	free_eq_pages(eq);
816 	return err;
817 }
818 
819 /**
820  * remove_eq - remove Event Queue
821  * @eq: the event queue
822  **/
823 static void remove_eq(struct hinic_eq *eq)
824 {
825 	hinic_set_msix_state(eq->hwif, eq->msix_entry.entry,
826 			     HINIC_MSIX_DISABLE);
827 	free_irq(eq->msix_entry.vector, eq);
828 
829 	if (eq->type == HINIC_AEQ) {
830 		struct hinic_eq_work *aeq_work = &eq->aeq_work;
831 
832 		cancel_work_sync(&aeq_work->work);
833 		/* clear aeq_len to avoid hw access host memory */
834 		hinic_hwif_write_reg(eq->hwif,
835 				     HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id), 0);
836 	} else if (eq->type == HINIC_CEQ) {
837 		tasklet_kill(&eq->ceq_tasklet);
838 		/* clear ceq_len to avoid hw access host memory */
839 		hinic_hwif_write_reg(eq->hwif,
840 				     HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id), 0);
841 	}
842 
843 	/* update cons_idx to avoid invalid interrupt */
844 	eq->cons_idx = hinic_hwif_read_reg(eq->hwif, EQ_PROD_IDX_REG_ADDR(eq));
845 	eq_update_ci(eq, EQ_NOT_ARMED);
846 
847 	free_eq_pages(eq);
848 }
849 
850 /**
851  * hinic_aeqs_init - initialize all the aeqs
852  * @aeqs: pointer to Async eqs of the chip
853  * @hwif: the HW interface of a PCI function device
854  * @num_aeqs: number of AEQs
855  * @q_len: number of EQ elements
856  * @page_size: the page size of the pages in the event queue
857  * @msix_entries: msix entries associated with the event queues
858  *
859  * Return 0 - Success, negative - Failure
860  **/
861 int hinic_aeqs_init(struct hinic_aeqs *aeqs, struct hinic_hwif *hwif,
862 		    int num_aeqs, u32 q_len, u32 page_size,
863 		    struct msix_entry *msix_entries)
864 {
865 	struct pci_dev *pdev = hwif->pdev;
866 	int err, i, q_id;
867 
868 	aeqs->workq = create_singlethread_workqueue(HINIC_EQS_WQ_NAME);
869 	if (!aeqs->workq)
870 		return -ENOMEM;
871 
872 	aeqs->hwif = hwif;
873 	aeqs->num_aeqs = num_aeqs;
874 
875 	for (q_id = 0; q_id < num_aeqs; q_id++) {
876 		err = init_eq(&aeqs->aeq[q_id], hwif, HINIC_AEQ, q_id, q_len,
877 			      page_size, msix_entries[q_id]);
878 		if (err) {
879 			dev_err(&pdev->dev, "Failed to init aeq %d\n", q_id);
880 			goto err_init_aeq;
881 		}
882 	}
883 
884 	return 0;
885 
886 err_init_aeq:
887 	for (i = 0; i < q_id; i++)
888 		remove_eq(&aeqs->aeq[i]);
889 
890 	destroy_workqueue(aeqs->workq);
891 	return err;
892 }
893 
894 /**
895  * hinic_aeqs_free - free all the aeqs
896  * @aeqs: pointer to Async eqs of the chip
897  **/
898 void hinic_aeqs_free(struct hinic_aeqs *aeqs)
899 {
900 	int q_id;
901 
902 	for (q_id = 0; q_id < aeqs->num_aeqs ; q_id++)
903 		remove_eq(&aeqs->aeq[q_id]);
904 
905 	destroy_workqueue(aeqs->workq);
906 }
907 
908 /**
909  * hinic_ceqs_init - init all the ceqs
910  * @ceqs: ceqs part of the chip
911  * @hwif: the hardware interface of a pci function device
912  * @num_ceqs: number of CEQs
913  * @q_len: number of EQ elements
914  * @page_size: the page size of the event queue
915  * @msix_entries: msix entries associated with the event queues
916  *
917  * Return 0 - Success, Negative - Failure
918  **/
919 int hinic_ceqs_init(struct hinic_ceqs *ceqs, struct hinic_hwif *hwif,
920 		    int num_ceqs, u32 q_len, u32 page_size,
921 		    struct msix_entry *msix_entries)
922 {
923 	struct pci_dev *pdev = hwif->pdev;
924 	int i, q_id, err;
925 
926 	ceqs->hwif = hwif;
927 	ceqs->num_ceqs = num_ceqs;
928 
929 	for (q_id = 0; q_id < num_ceqs; q_id++) {
930 		ceqs->ceq[q_id].hwdev = ceqs->hwdev;
931 		err = init_eq(&ceqs->ceq[q_id], hwif, HINIC_CEQ, q_id, q_len,
932 			      page_size, msix_entries[q_id]);
933 		if (err) {
934 			dev_err(&pdev->dev, "Failed to init ceq %d\n", q_id);
935 			goto err_init_ceq;
936 		}
937 	}
938 
939 	return 0;
940 
941 err_init_ceq:
942 	for (i = 0; i < q_id; i++)
943 		remove_eq(&ceqs->ceq[i]);
944 
945 	return err;
946 }
947 
948 /**
949  * hinic_ceqs_free - free all the ceqs
950  * @ceqs: ceqs part of the chip
951  **/
952 void hinic_ceqs_free(struct hinic_ceqs *ceqs)
953 {
954 	int q_id;
955 
956 	for (q_id = 0; q_id < ceqs->num_ceqs; q_id++)
957 		remove_eq(&ceqs->ceq[q_id]);
958 }
959 
960 void hinic_dump_ceq_info(struct hinic_hwdev *hwdev)
961 {
962 	struct hinic_eq *eq = NULL;
963 	u32 addr, ci, pi;
964 	int q_id;
965 
966 	for (q_id = 0; q_id < hwdev->func_to_io.ceqs.num_ceqs; q_id++) {
967 		eq = &hwdev->func_to_io.ceqs.ceq[q_id];
968 		addr = EQ_CONS_IDX_REG_ADDR(eq);
969 		ci = hinic_hwif_read_reg(hwdev->hwif, addr);
970 		addr = EQ_PROD_IDX_REG_ADDR(eq);
971 		pi = hinic_hwif_read_reg(hwdev->hwif, addr);
972 		dev_err(&hwdev->hwif->pdev->dev, "Ceq id: %d, ci: 0x%08x, sw_ci: 0x%08x, pi: 0x%x, tasklet_state: 0x%lx, wrap: %d, ceqe: 0x%x\n",
973 			q_id, ci, eq->cons_idx, pi,
974 			eq->ceq_tasklet.state,
975 			eq->wrapped, be32_to_cpu(*(__be32 *)(GET_CURR_CEQ_ELEM(eq))));
976 	}
977 }
978 
979 void hinic_dump_aeq_info(struct hinic_hwdev *hwdev)
980 {
981 	struct hinic_aeq_elem *aeqe_pos = NULL;
982 	struct hinic_eq *eq = NULL;
983 	u32 addr, ci, pi;
984 	int q_id;
985 
986 	for (q_id = 0; q_id < hwdev->aeqs.num_aeqs; q_id++) {
987 		eq = &hwdev->aeqs.aeq[q_id];
988 		addr = EQ_CONS_IDX_REG_ADDR(eq);
989 		ci = hinic_hwif_read_reg(hwdev->hwif, addr);
990 		addr = EQ_PROD_IDX_REG_ADDR(eq);
991 		pi = hinic_hwif_read_reg(hwdev->hwif, addr);
992 		aeqe_pos = GET_CURR_AEQ_ELEM(eq);
993 		dev_err(&hwdev->hwif->pdev->dev, "Aeq id: %d, ci: 0x%08x, pi: 0x%x, work_state: 0x%x, wrap: %d, desc: 0x%x\n",
994 			q_id, ci, pi, work_busy(&eq->aeq_work.work),
995 			eq->wrapped, be32_to_cpu(aeqe_pos->desc));
996 	}
997 }
998