1 /*
2  * drivers/net/ethernet/mellanox/mlxsw/pci.c
3  * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
4  * Copyright (c) 2015 Jiri Pirko <jiri@mellanox.com>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are met:
8  *
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  * 3. Neither the names of the copyright holders nor the names of its
15  *    contributors may be used to endorse or promote products derived from
16  *    this software without specific prior written permission.
17  *
18  * Alternatively, this software may be distributed under the terms of the
19  * GNU General Public License ("GPL") version 2 as published by the Free
20  * Software Foundation.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32  * POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <linux/kernel.h>
36 #include <linux/module.h>
37 #include <linux/export.h>
38 #include <linux/err.h>
39 #include <linux/device.h>
40 #include <linux/pci.h>
41 #include <linux/interrupt.h>
42 #include <linux/wait.h>
43 #include <linux/types.h>
44 #include <linux/skbuff.h>
45 #include <linux/if_vlan.h>
46 #include <linux/log2.h>
47 #include <linux/debugfs.h>
48 #include <linux/seq_file.h>
49 #include <linux/string.h>
50 
51 #include "pci.h"
52 #include "core.h"
53 #include "cmd.h"
54 #include "port.h"
55 
56 static const char mlxsw_pci_driver_name[] = "mlxsw_pci";
57 
58 static const struct pci_device_id mlxsw_pci_id_table[] = {
59 	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SWITCHX2), 0},
60 	{0, }
61 };
62 
63 static struct dentry *mlxsw_pci_dbg_root;
64 
65 static const char *mlxsw_pci_device_kind_get(const struct pci_device_id *id)
66 {
67 	switch (id->device) {
68 	case PCI_DEVICE_ID_MELLANOX_SWITCHX2:
69 		return MLXSW_DEVICE_KIND_SWITCHX2;
70 	default:
71 		BUG();
72 	}
73 }
74 
75 #define mlxsw_pci_write32(mlxsw_pci, reg, val) \
76 	iowrite32be(val, (mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
77 #define mlxsw_pci_read32(mlxsw_pci, reg) \
78 	ioread32be((mlxsw_pci)->hw_addr + (MLXSW_PCI_ ## reg))
79 
80 enum mlxsw_pci_queue_type {
81 	MLXSW_PCI_QUEUE_TYPE_SDQ,
82 	MLXSW_PCI_QUEUE_TYPE_RDQ,
83 	MLXSW_PCI_QUEUE_TYPE_CQ,
84 	MLXSW_PCI_QUEUE_TYPE_EQ,
85 };
86 
87 static const char *mlxsw_pci_queue_type_str(enum mlxsw_pci_queue_type q_type)
88 {
89 	switch (q_type) {
90 	case MLXSW_PCI_QUEUE_TYPE_SDQ:
91 		return "sdq";
92 	case MLXSW_PCI_QUEUE_TYPE_RDQ:
93 		return "rdq";
94 	case MLXSW_PCI_QUEUE_TYPE_CQ:
95 		return "cq";
96 	case MLXSW_PCI_QUEUE_TYPE_EQ:
97 		return "eq";
98 	}
99 	BUG();
100 }
101 
102 #define MLXSW_PCI_QUEUE_TYPE_COUNT	4
103 
104 static const u16 mlxsw_pci_doorbell_type_offset[] = {
105 	MLXSW_PCI_DOORBELL_SDQ_OFFSET,	/* for type MLXSW_PCI_QUEUE_TYPE_SDQ */
106 	MLXSW_PCI_DOORBELL_RDQ_OFFSET,	/* for type MLXSW_PCI_QUEUE_TYPE_RDQ */
107 	MLXSW_PCI_DOORBELL_CQ_OFFSET,	/* for type MLXSW_PCI_QUEUE_TYPE_CQ */
108 	MLXSW_PCI_DOORBELL_EQ_OFFSET,	/* for type MLXSW_PCI_QUEUE_TYPE_EQ */
109 };
110 
111 static const u16 mlxsw_pci_doorbell_arm_type_offset[] = {
112 	0, /* unused */
113 	0, /* unused */
114 	MLXSW_PCI_DOORBELL_ARM_CQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_CQ */
115 	MLXSW_PCI_DOORBELL_ARM_EQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_EQ */
116 };
117 
118 struct mlxsw_pci_mem_item {
119 	char *buf;
120 	dma_addr_t mapaddr;
121 	size_t size;
122 };
123 
124 struct mlxsw_pci_queue_elem_info {
125 	char *elem; /* pointer to actual dma mapped element mem chunk */
126 	union {
127 		struct {
128 			struct sk_buff *skb;
129 		} sdq;
130 		struct {
131 			struct sk_buff *skb;
132 		} rdq;
133 	} u;
134 };
135 
136 struct mlxsw_pci_queue {
137 	spinlock_t lock; /* for queue accesses */
138 	struct mlxsw_pci_mem_item mem_item;
139 	struct mlxsw_pci_queue_elem_info *elem_info;
140 	u16 producer_counter;
141 	u16 consumer_counter;
142 	u16 count; /* number of elements in queue */
143 	u8 num; /* queue number */
144 	u8 elem_size; /* size of one element */
145 	enum mlxsw_pci_queue_type type;
146 	struct tasklet_struct tasklet; /* queue processing tasklet */
147 	struct mlxsw_pci *pci;
148 	union {
149 		struct {
150 			u32 comp_sdq_count;
151 			u32 comp_rdq_count;
152 		} cq;
153 		struct {
154 			u32 ev_cmd_count;
155 			u32 ev_comp_count;
156 			u32 ev_other_count;
157 		} eq;
158 	} u;
159 };
160 
161 struct mlxsw_pci_queue_type_group {
162 	struct mlxsw_pci_queue *q;
163 	u8 count; /* number of queues in group */
164 };
165 
166 struct mlxsw_pci {
167 	struct pci_dev *pdev;
168 	u8 __iomem *hw_addr;
169 	struct mlxsw_pci_queue_type_group queues[MLXSW_PCI_QUEUE_TYPE_COUNT];
170 	u32 doorbell_offset;
171 	struct msix_entry msix_entry;
172 	struct mlxsw_core *core;
173 	struct {
174 		u16 num_pages;
175 		struct mlxsw_pci_mem_item *items;
176 	} fw_area;
177 	struct {
178 		struct mlxsw_pci_mem_item out_mbox;
179 		struct mlxsw_pci_mem_item in_mbox;
180 		struct mutex lock; /* Lock access to command registers */
181 		bool nopoll;
182 		wait_queue_head_t wait;
183 		bool wait_done;
184 		struct {
185 			u8 status;
186 			u64 out_param;
187 		} comp;
188 	} cmd;
189 	struct mlxsw_bus_info bus_info;
190 	struct dentry *dbg_dir;
191 };
192 
193 static void mlxsw_pci_queue_tasklet_schedule(struct mlxsw_pci_queue *q)
194 {
195 	tasklet_schedule(&q->tasklet);
196 }
197 
198 static char *__mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q,
199 					size_t elem_size, int elem_index)
200 {
201 	return q->mem_item.buf + (elem_size * elem_index);
202 }
203 
204 static struct mlxsw_pci_queue_elem_info *
205 mlxsw_pci_queue_elem_info_get(struct mlxsw_pci_queue *q, int elem_index)
206 {
207 	return &q->elem_info[elem_index];
208 }
209 
210 static struct mlxsw_pci_queue_elem_info *
211 mlxsw_pci_queue_elem_info_producer_get(struct mlxsw_pci_queue *q)
212 {
213 	int index = q->producer_counter & (q->count - 1);
214 
215 	if ((q->producer_counter - q->consumer_counter) == q->count)
216 		return NULL;
217 	return mlxsw_pci_queue_elem_info_get(q, index);
218 }
219 
220 static struct mlxsw_pci_queue_elem_info *
221 mlxsw_pci_queue_elem_info_consumer_get(struct mlxsw_pci_queue *q)
222 {
223 	int index = q->consumer_counter & (q->count - 1);
224 
225 	return mlxsw_pci_queue_elem_info_get(q, index);
226 }
227 
228 static char *mlxsw_pci_queue_elem_get(struct mlxsw_pci_queue *q, int elem_index)
229 {
230 	return mlxsw_pci_queue_elem_info_get(q, elem_index)->elem;
231 }
232 
233 static bool mlxsw_pci_elem_hw_owned(struct mlxsw_pci_queue *q, bool owner_bit)
234 {
235 	return owner_bit != !!(q->consumer_counter & q->count);
236 }
237 
238 static char *mlxsw_pci_queue_sw_elem_get(struct mlxsw_pci_queue *q,
239 					 u32 (*get_elem_owner_func)(char *))
240 {
241 	struct mlxsw_pci_queue_elem_info *elem_info;
242 	char *elem;
243 	bool owner_bit;
244 
245 	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
246 	elem = elem_info->elem;
247 	owner_bit = get_elem_owner_func(elem);
248 	if (mlxsw_pci_elem_hw_owned(q, owner_bit))
249 		return NULL;
250 	q->consumer_counter++;
251 	rmb(); /* make sure we read owned bit before the rest of elem */
252 	return elem;
253 }
254 
255 static struct mlxsw_pci_queue_type_group *
256 mlxsw_pci_queue_type_group_get(struct mlxsw_pci *mlxsw_pci,
257 			       enum mlxsw_pci_queue_type q_type)
258 {
259 	return &mlxsw_pci->queues[q_type];
260 }
261 
262 static u8 __mlxsw_pci_queue_count(struct mlxsw_pci *mlxsw_pci,
263 				  enum mlxsw_pci_queue_type q_type)
264 {
265 	struct mlxsw_pci_queue_type_group *queue_group;
266 
267 	queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_type);
268 	return queue_group->count;
269 }
270 
271 static u8 mlxsw_pci_sdq_count(struct mlxsw_pci *mlxsw_pci)
272 {
273 	return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_SDQ);
274 }
275 
276 static u8 mlxsw_pci_rdq_count(struct mlxsw_pci *mlxsw_pci)
277 {
278 	return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_RDQ);
279 }
280 
281 static u8 mlxsw_pci_cq_count(struct mlxsw_pci *mlxsw_pci)
282 {
283 	return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ);
284 }
285 
286 static u8 mlxsw_pci_eq_count(struct mlxsw_pci *mlxsw_pci)
287 {
288 	return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ);
289 }
290 
291 static struct mlxsw_pci_queue *
292 __mlxsw_pci_queue_get(struct mlxsw_pci *mlxsw_pci,
293 		      enum mlxsw_pci_queue_type q_type, u8 q_num)
294 {
295 	return &mlxsw_pci->queues[q_type].q[q_num];
296 }
297 
298 static struct mlxsw_pci_queue *mlxsw_pci_sdq_get(struct mlxsw_pci *mlxsw_pci,
299 						 u8 q_num)
300 {
301 	return __mlxsw_pci_queue_get(mlxsw_pci,
302 				     MLXSW_PCI_QUEUE_TYPE_SDQ, q_num);
303 }
304 
305 static struct mlxsw_pci_queue *mlxsw_pci_rdq_get(struct mlxsw_pci *mlxsw_pci,
306 						 u8 q_num)
307 {
308 	return __mlxsw_pci_queue_get(mlxsw_pci,
309 				     MLXSW_PCI_QUEUE_TYPE_RDQ, q_num);
310 }
311 
312 static struct mlxsw_pci_queue *mlxsw_pci_cq_get(struct mlxsw_pci *mlxsw_pci,
313 						u8 q_num)
314 {
315 	return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ, q_num);
316 }
317 
318 static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci,
319 						u8 q_num)
320 {
321 	return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ, q_num);
322 }
323 
324 static void __mlxsw_pci_queue_doorbell_set(struct mlxsw_pci *mlxsw_pci,
325 					   struct mlxsw_pci_queue *q,
326 					   u16 val)
327 {
328 	mlxsw_pci_write32(mlxsw_pci,
329 			  DOORBELL(mlxsw_pci->doorbell_offset,
330 				   mlxsw_pci_doorbell_type_offset[q->type],
331 				   q->num), val);
332 }
333 
334 static void __mlxsw_pci_queue_doorbell_arm_set(struct mlxsw_pci *mlxsw_pci,
335 					       struct mlxsw_pci_queue *q,
336 					       u16 val)
337 {
338 	mlxsw_pci_write32(mlxsw_pci,
339 			  DOORBELL(mlxsw_pci->doorbell_offset,
340 				   mlxsw_pci_doorbell_arm_type_offset[q->type],
341 				   q->num), val);
342 }
343 
344 static void mlxsw_pci_queue_doorbell_producer_ring(struct mlxsw_pci *mlxsw_pci,
345 						   struct mlxsw_pci_queue *q)
346 {
347 	wmb(); /* ensure all writes are done before we ring a bell */
348 	__mlxsw_pci_queue_doorbell_set(mlxsw_pci, q, q->producer_counter);
349 }
350 
351 static void mlxsw_pci_queue_doorbell_consumer_ring(struct mlxsw_pci *mlxsw_pci,
352 						   struct mlxsw_pci_queue *q)
353 {
354 	wmb(); /* ensure all writes are done before we ring a bell */
355 	__mlxsw_pci_queue_doorbell_set(mlxsw_pci, q,
356 				       q->consumer_counter + q->count);
357 }
358 
359 static void
360 mlxsw_pci_queue_doorbell_arm_consumer_ring(struct mlxsw_pci *mlxsw_pci,
361 					   struct mlxsw_pci_queue *q)
362 {
363 	wmb(); /* ensure all writes are done before we ring a bell */
364 	__mlxsw_pci_queue_doorbell_arm_set(mlxsw_pci, q, q->consumer_counter);
365 }
366 
367 static dma_addr_t __mlxsw_pci_queue_page_get(struct mlxsw_pci_queue *q,
368 					     int page_index)
369 {
370 	return q->mem_item.mapaddr + MLXSW_PCI_PAGE_SIZE * page_index;
371 }
372 
373 static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
374 			      struct mlxsw_pci_queue *q)
375 {
376 	int i;
377 	int err;
378 
379 	q->producer_counter = 0;
380 	q->consumer_counter = 0;
381 
382 	/* Set CQ of same number of this SDQ. */
383 	mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num);
384 	mlxsw_cmd_mbox_sw2hw_dq_sdq_tclass_set(mbox, 7);
385 	mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */
386 	for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
387 		dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
388 
389 		mlxsw_cmd_mbox_sw2hw_dq_pa_set(mbox, i, mapaddr);
390 	}
391 
392 	err = mlxsw_cmd_sw2hw_sdq(mlxsw_pci->core, mbox, q->num);
393 	if (err)
394 		return err;
395 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
396 	return 0;
397 }
398 
399 static void mlxsw_pci_sdq_fini(struct mlxsw_pci *mlxsw_pci,
400 			       struct mlxsw_pci_queue *q)
401 {
402 	mlxsw_cmd_hw2sw_sdq(mlxsw_pci->core, q->num);
403 }
404 
405 static int mlxsw_pci_sdq_dbg_read(struct seq_file *file, void *data)
406 {
407 	struct mlxsw_pci *mlxsw_pci = dev_get_drvdata(file->private);
408 	struct mlxsw_pci_queue *q;
409 	int i;
410 	static const char hdr[] =
411 		"NUM PROD_COUNT CONS_COUNT COUNT\n";
412 
413 	seq_printf(file, hdr);
414 	for (i = 0; i < mlxsw_pci_sdq_count(mlxsw_pci); i++) {
415 		q = mlxsw_pci_sdq_get(mlxsw_pci, i);
416 		spin_lock_bh(&q->lock);
417 		seq_printf(file, "%3d %10d %10d %5d\n",
418 			   i, q->producer_counter, q->consumer_counter,
419 			   q->count);
420 		spin_unlock_bh(&q->lock);
421 	}
422 	return 0;
423 }
424 
425 static int mlxsw_pci_wqe_frag_map(struct mlxsw_pci *mlxsw_pci, char *wqe,
426 				  int index, char *frag_data, size_t frag_len,
427 				  int direction)
428 {
429 	struct pci_dev *pdev = mlxsw_pci->pdev;
430 	dma_addr_t mapaddr;
431 
432 	mapaddr = pci_map_single(pdev, frag_data, frag_len, direction);
433 	if (unlikely(pci_dma_mapping_error(pdev, mapaddr))) {
434 		if (net_ratelimit())
435 			dev_err(&pdev->dev, "failed to dma map tx frag\n");
436 		return -EIO;
437 	}
438 	mlxsw_pci_wqe_address_set(wqe, index, mapaddr);
439 	mlxsw_pci_wqe_byte_count_set(wqe, index, frag_len);
440 	return 0;
441 }
442 
443 static void mlxsw_pci_wqe_frag_unmap(struct mlxsw_pci *mlxsw_pci, char *wqe,
444 				     int index, int direction)
445 {
446 	struct pci_dev *pdev = mlxsw_pci->pdev;
447 	size_t frag_len = mlxsw_pci_wqe_byte_count_get(wqe, index);
448 	dma_addr_t mapaddr = mlxsw_pci_wqe_address_get(wqe, index);
449 
450 	if (!frag_len)
451 		return;
452 	pci_unmap_single(pdev, mapaddr, frag_len, direction);
453 }
454 
455 static int mlxsw_pci_rdq_skb_alloc(struct mlxsw_pci *mlxsw_pci,
456 				   struct mlxsw_pci_queue_elem_info *elem_info)
457 {
458 	size_t buf_len = MLXSW_PORT_MAX_MTU;
459 	char *wqe = elem_info->elem;
460 	struct sk_buff *skb;
461 	int err;
462 
463 	elem_info->u.rdq.skb = NULL;
464 	skb = netdev_alloc_skb_ip_align(NULL, buf_len);
465 	if (!skb)
466 		return -ENOMEM;
467 
468 	/* Assume that wqe was previously zeroed. */
469 
470 	err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,
471 				     buf_len, DMA_FROM_DEVICE);
472 	if (err)
473 		goto err_frag_map;
474 
475 	elem_info->u.rdq.skb = skb;
476 	return 0;
477 
478 err_frag_map:
479 	dev_kfree_skb_any(skb);
480 	return err;
481 }
482 
483 static void mlxsw_pci_rdq_skb_free(struct mlxsw_pci *mlxsw_pci,
484 				   struct mlxsw_pci_queue_elem_info *elem_info)
485 {
486 	struct sk_buff *skb;
487 	char *wqe;
488 
489 	skb = elem_info->u.rdq.skb;
490 	wqe = elem_info->elem;
491 
492 	mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE);
493 	dev_kfree_skb_any(skb);
494 }
495 
496 static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
497 			      struct mlxsw_pci_queue *q)
498 {
499 	struct mlxsw_pci_queue_elem_info *elem_info;
500 	int i;
501 	int err;
502 
503 	q->producer_counter = 0;
504 	q->consumer_counter = 0;
505 
506 	/* Set CQ of same number of this RDQ with base
507 	 * above MLXSW_PCI_SDQS_MAX as the lower ones are assigned to SDQs.
508 	 */
509 	mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num + MLXSW_PCI_SDQS_COUNT);
510 	mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */
511 	for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
512 		dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
513 
514 		mlxsw_cmd_mbox_sw2hw_dq_pa_set(mbox, i, mapaddr);
515 	}
516 
517 	err = mlxsw_cmd_sw2hw_rdq(mlxsw_pci->core, mbox, q->num);
518 	if (err)
519 		return err;
520 
521 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
522 
523 	for (i = 0; i < q->count; i++) {
524 		elem_info = mlxsw_pci_queue_elem_info_producer_get(q);
525 		BUG_ON(!elem_info);
526 		err = mlxsw_pci_rdq_skb_alloc(mlxsw_pci, elem_info);
527 		if (err)
528 			goto rollback;
529 		/* Everything is set up, ring doorbell to pass elem to HW */
530 		q->producer_counter++;
531 		mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
532 	}
533 
534 	return 0;
535 
536 rollback:
537 	for (i--; i >= 0; i--) {
538 		elem_info = mlxsw_pci_queue_elem_info_get(q, i);
539 		mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info);
540 	}
541 	mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);
542 
543 	return err;
544 }
545 
546 static void mlxsw_pci_rdq_fini(struct mlxsw_pci *mlxsw_pci,
547 			       struct mlxsw_pci_queue *q)
548 {
549 	struct mlxsw_pci_queue_elem_info *elem_info;
550 	int i;
551 
552 	mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num);
553 	for (i = 0; i < q->count; i++) {
554 		elem_info = mlxsw_pci_queue_elem_info_get(q, i);
555 		mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info);
556 	}
557 }
558 
559 static int mlxsw_pci_rdq_dbg_read(struct seq_file *file, void *data)
560 {
561 	struct mlxsw_pci *mlxsw_pci = dev_get_drvdata(file->private);
562 	struct mlxsw_pci_queue *q;
563 	int i;
564 	static const char hdr[] =
565 		"NUM PROD_COUNT CONS_COUNT COUNT\n";
566 
567 	seq_printf(file, hdr);
568 	for (i = 0; i < mlxsw_pci_rdq_count(mlxsw_pci); i++) {
569 		q = mlxsw_pci_rdq_get(mlxsw_pci, i);
570 		spin_lock_bh(&q->lock);
571 		seq_printf(file, "%3d %10d %10d %5d\n",
572 			   i, q->producer_counter, q->consumer_counter,
573 			   q->count);
574 		spin_unlock_bh(&q->lock);
575 	}
576 	return 0;
577 }
578 
579 static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
580 			     struct mlxsw_pci_queue *q)
581 {
582 	int i;
583 	int err;
584 
585 	q->consumer_counter = 0;
586 
587 	for (i = 0; i < q->count; i++) {
588 		char *elem = mlxsw_pci_queue_elem_get(q, i);
589 
590 		mlxsw_pci_cqe_owner_set(elem, 1);
591 	}
592 
593 	mlxsw_cmd_mbox_sw2hw_cq_cv_set(mbox, 0); /* CQE ver 0 */
594 	mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM);
595 	mlxsw_cmd_mbox_sw2hw_cq_oi_set(mbox, 0);
596 	mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0);
597 	mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count));
598 	for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
599 		dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
600 
601 		mlxsw_cmd_mbox_sw2hw_cq_pa_set(mbox, i, mapaddr);
602 	}
603 	err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num);
604 	if (err)
605 		return err;
606 	mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
607 	mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
608 	return 0;
609 }
610 
611 static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci,
612 			      struct mlxsw_pci_queue *q)
613 {
614 	mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num);
615 }
616 
617 static int mlxsw_pci_cq_dbg_read(struct seq_file *file, void *data)
618 {
619 	struct mlxsw_pci *mlxsw_pci = dev_get_drvdata(file->private);
620 
621 	struct mlxsw_pci_queue *q;
622 	int i;
623 	static const char hdr[] =
624 		"NUM CONS_INDEX  SDQ_COUNT  RDQ_COUNT COUNT\n";
625 
626 	seq_printf(file, hdr);
627 	for (i = 0; i < mlxsw_pci_cq_count(mlxsw_pci); i++) {
628 		q = mlxsw_pci_cq_get(mlxsw_pci, i);
629 		spin_lock_bh(&q->lock);
630 		seq_printf(file, "%3d %10d %10d %10d %5d\n",
631 			   i, q->consumer_counter, q->u.cq.comp_sdq_count,
632 			   q->u.cq.comp_rdq_count, q->count);
633 		spin_unlock_bh(&q->lock);
634 	}
635 	return 0;
636 }
637 
638 static void mlxsw_pci_cqe_sdq_handle(struct mlxsw_pci *mlxsw_pci,
639 				     struct mlxsw_pci_queue *q,
640 				     u16 consumer_counter_limit,
641 				     char *cqe)
642 {
643 	struct pci_dev *pdev = mlxsw_pci->pdev;
644 	struct mlxsw_pci_queue_elem_info *elem_info;
645 	char *wqe;
646 	struct sk_buff *skb;
647 	int i;
648 
649 	spin_lock(&q->lock);
650 	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
651 	skb = elem_info->u.sdq.skb;
652 	wqe = elem_info->elem;
653 	for (i = 0; i < MLXSW_PCI_WQE_SG_ENTRIES; i++)
654 		mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE);
655 	dev_kfree_skb_any(skb);
656 	elem_info->u.sdq.skb = NULL;
657 
658 	if (q->consumer_counter++ != consumer_counter_limit)
659 		dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in SDQ\n");
660 	spin_unlock(&q->lock);
661 }
662 
663 static void mlxsw_pci_cqe_rdq_handle(struct mlxsw_pci *mlxsw_pci,
664 				     struct mlxsw_pci_queue *q,
665 				     u16 consumer_counter_limit,
666 				     char *cqe)
667 {
668 	struct pci_dev *pdev = mlxsw_pci->pdev;
669 	struct mlxsw_pci_queue_elem_info *elem_info;
670 	char *wqe;
671 	struct sk_buff *skb;
672 	struct mlxsw_rx_info rx_info;
673 	u16 byte_count;
674 	int err;
675 
676 	elem_info = mlxsw_pci_queue_elem_info_consumer_get(q);
677 	skb = elem_info->u.sdq.skb;
678 	if (!skb)
679 		return;
680 	wqe = elem_info->elem;
681 	mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, 0, DMA_FROM_DEVICE);
682 
683 	if (q->consumer_counter++ != consumer_counter_limit)
684 		dev_dbg_ratelimited(&pdev->dev, "Consumer counter does not match limit in RDQ\n");
685 
686 	/* We do not support lag now */
687 	if (mlxsw_pci_cqe_lag_get(cqe))
688 		goto drop;
689 
690 	rx_info.sys_port = mlxsw_pci_cqe_system_port_get(cqe);
691 	rx_info.trap_id = mlxsw_pci_cqe_trap_id_get(cqe);
692 
693 	byte_count = mlxsw_pci_cqe_byte_count_get(cqe);
694 	if (mlxsw_pci_cqe_crc_get(cqe))
695 		byte_count -= ETH_FCS_LEN;
696 	skb_put(skb, byte_count);
697 	mlxsw_core_skb_receive(mlxsw_pci->core, skb, &rx_info);
698 
699 put_new_skb:
700 	memset(wqe, 0, q->elem_size);
701 	err = mlxsw_pci_rdq_skb_alloc(mlxsw_pci, elem_info);
702 	if (err && net_ratelimit())
703 		dev_dbg(&pdev->dev, "Failed to alloc skb for RDQ\n");
704 	/* Everything is set up, ring doorbell to pass elem to HW */
705 	q->producer_counter++;
706 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
707 	return;
708 
709 drop:
710 	dev_kfree_skb_any(skb);
711 	goto put_new_skb;
712 }
713 
714 static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q)
715 {
716 	return mlxsw_pci_queue_sw_elem_get(q, mlxsw_pci_cqe_owner_get);
717 }
718 
719 static void mlxsw_pci_cq_tasklet(unsigned long data)
720 {
721 	struct mlxsw_pci_queue *q = (struct mlxsw_pci_queue *) data;
722 	struct mlxsw_pci *mlxsw_pci = q->pci;
723 	char *cqe;
724 	int items = 0;
725 	int credits = q->count >> 1;
726 
727 	while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) {
728 		u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe);
729 		u8 sendq = mlxsw_pci_cqe_sr_get(cqe);
730 		u8 dqn = mlxsw_pci_cqe_dqn_get(cqe);
731 
732 		if (sendq) {
733 			struct mlxsw_pci_queue *sdq;
734 
735 			sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn);
736 			mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq,
737 						 wqe_counter, cqe);
738 			q->u.cq.comp_sdq_count++;
739 		} else {
740 			struct mlxsw_pci_queue *rdq;
741 
742 			rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn);
743 			mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq,
744 						 wqe_counter, cqe);
745 			q->u.cq.comp_rdq_count++;
746 		}
747 		if (++items == credits)
748 			break;
749 	}
750 	if (items) {
751 		mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
752 		mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
753 	}
754 }
755 
756 static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
757 			     struct mlxsw_pci_queue *q)
758 {
759 	int i;
760 	int err;
761 
762 	q->consumer_counter = 0;
763 
764 	for (i = 0; i < q->count; i++) {
765 		char *elem = mlxsw_pci_queue_elem_get(q, i);
766 
767 		mlxsw_pci_eqe_owner_set(elem, 1);
768 	}
769 
770 	mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */
771 	mlxsw_cmd_mbox_sw2hw_eq_oi_set(mbox, 0);
772 	mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */
773 	mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count));
774 	for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) {
775 		dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i);
776 
777 		mlxsw_cmd_mbox_sw2hw_eq_pa_set(mbox, i, mapaddr);
778 	}
779 	err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num);
780 	if (err)
781 		return err;
782 	mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
783 	mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
784 	return 0;
785 }
786 
787 static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci,
788 			      struct mlxsw_pci_queue *q)
789 {
790 	mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num);
791 }
792 
793 static int mlxsw_pci_eq_dbg_read(struct seq_file *file, void *data)
794 {
795 	struct mlxsw_pci *mlxsw_pci = dev_get_drvdata(file->private);
796 	struct mlxsw_pci_queue *q;
797 	int i;
798 	static const char hdr[] =
799 		"NUM CONS_COUNT     EV_CMD    EV_COMP   EV_OTHER COUNT\n";
800 
801 	seq_printf(file, hdr);
802 	for (i = 0; i < mlxsw_pci_eq_count(mlxsw_pci); i++) {
803 		q = mlxsw_pci_eq_get(mlxsw_pci, i);
804 		spin_lock_bh(&q->lock);
805 		seq_printf(file, "%3d %10d %10d %10d %10d %5d\n",
806 			   i, q->consumer_counter, q->u.eq.ev_cmd_count,
807 			   q->u.eq.ev_comp_count, q->u.eq.ev_other_count,
808 			   q->count);
809 		spin_unlock_bh(&q->lock);
810 	}
811 	return 0;
812 }
813 
814 static void mlxsw_pci_eq_cmd_event(struct mlxsw_pci *mlxsw_pci, char *eqe)
815 {
816 	mlxsw_pci->cmd.comp.status = mlxsw_pci_eqe_cmd_status_get(eqe);
817 	mlxsw_pci->cmd.comp.out_param =
818 		((u64) mlxsw_pci_eqe_cmd_out_param_h_get(eqe)) << 32 |
819 		mlxsw_pci_eqe_cmd_out_param_l_get(eqe);
820 	mlxsw_pci->cmd.wait_done = true;
821 	wake_up(&mlxsw_pci->cmd.wait);
822 }
823 
824 static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q)
825 {
826 	return mlxsw_pci_queue_sw_elem_get(q, mlxsw_pci_eqe_owner_get);
827 }
828 
829 static void mlxsw_pci_eq_tasklet(unsigned long data)
830 {
831 	struct mlxsw_pci_queue *q = (struct mlxsw_pci_queue *) data;
832 	struct mlxsw_pci *mlxsw_pci = q->pci;
833 	unsigned long active_cqns[BITS_TO_LONGS(MLXSW_PCI_CQS_COUNT)];
834 	char *eqe;
835 	u8 cqn;
836 	bool cq_handle = false;
837 	int items = 0;
838 	int credits = q->count >> 1;
839 
840 	memset(&active_cqns, 0, sizeof(active_cqns));
841 
842 	while ((eqe = mlxsw_pci_eq_sw_eqe_get(q))) {
843 		u8 event_type = mlxsw_pci_eqe_event_type_get(eqe);
844 
845 		switch (event_type) {
846 		case MLXSW_PCI_EQE_EVENT_TYPE_CMD:
847 			mlxsw_pci_eq_cmd_event(mlxsw_pci, eqe);
848 			q->u.eq.ev_cmd_count++;
849 			break;
850 		case MLXSW_PCI_EQE_EVENT_TYPE_COMP:
851 			cqn = mlxsw_pci_eqe_cqn_get(eqe);
852 			set_bit(cqn, active_cqns);
853 			cq_handle = true;
854 			q->u.eq.ev_comp_count++;
855 			break;
856 		default:
857 			q->u.eq.ev_other_count++;
858 		}
859 		if (++items == credits)
860 			break;
861 	}
862 	if (items) {
863 		mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q);
864 		mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q);
865 	}
866 
867 	if (!cq_handle)
868 		return;
869 	for_each_set_bit(cqn, active_cqns, MLXSW_PCI_CQS_COUNT) {
870 		q = mlxsw_pci_cq_get(mlxsw_pci, cqn);
871 		mlxsw_pci_queue_tasklet_schedule(q);
872 	}
873 }
874 
875 struct mlxsw_pci_queue_ops {
876 	const char *name;
877 	enum mlxsw_pci_queue_type type;
878 	int (*init)(struct mlxsw_pci *mlxsw_pci, char *mbox,
879 		    struct mlxsw_pci_queue *q);
880 	void (*fini)(struct mlxsw_pci *mlxsw_pci,
881 		     struct mlxsw_pci_queue *q);
882 	void (*tasklet)(unsigned long data);
883 	int (*dbg_read)(struct seq_file *s, void *data);
884 	u16 elem_count;
885 	u8 elem_size;
886 };
887 
888 static const struct mlxsw_pci_queue_ops mlxsw_pci_sdq_ops = {
889 	.type		= MLXSW_PCI_QUEUE_TYPE_SDQ,
890 	.init		= mlxsw_pci_sdq_init,
891 	.fini		= mlxsw_pci_sdq_fini,
892 	.dbg_read	= mlxsw_pci_sdq_dbg_read,
893 	.elem_count	= MLXSW_PCI_WQE_COUNT,
894 	.elem_size	= MLXSW_PCI_WQE_SIZE,
895 };
896 
897 static const struct mlxsw_pci_queue_ops mlxsw_pci_rdq_ops = {
898 	.type		= MLXSW_PCI_QUEUE_TYPE_RDQ,
899 	.init		= mlxsw_pci_rdq_init,
900 	.fini		= mlxsw_pci_rdq_fini,
901 	.dbg_read	= mlxsw_pci_rdq_dbg_read,
902 	.elem_count	= MLXSW_PCI_WQE_COUNT,
903 	.elem_size	= MLXSW_PCI_WQE_SIZE
904 };
905 
906 static const struct mlxsw_pci_queue_ops mlxsw_pci_cq_ops = {
907 	.type		= MLXSW_PCI_QUEUE_TYPE_CQ,
908 	.init		= mlxsw_pci_cq_init,
909 	.fini		= mlxsw_pci_cq_fini,
910 	.tasklet	= mlxsw_pci_cq_tasklet,
911 	.dbg_read	= mlxsw_pci_cq_dbg_read,
912 	.elem_count	= MLXSW_PCI_CQE_COUNT,
913 	.elem_size	= MLXSW_PCI_CQE_SIZE
914 };
915 
916 static const struct mlxsw_pci_queue_ops mlxsw_pci_eq_ops = {
917 	.type		= MLXSW_PCI_QUEUE_TYPE_EQ,
918 	.init		= mlxsw_pci_eq_init,
919 	.fini		= mlxsw_pci_eq_fini,
920 	.tasklet	= mlxsw_pci_eq_tasklet,
921 	.dbg_read	= mlxsw_pci_eq_dbg_read,
922 	.elem_count	= MLXSW_PCI_EQE_COUNT,
923 	.elem_size	= MLXSW_PCI_EQE_SIZE
924 };
925 
926 static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
927 				const struct mlxsw_pci_queue_ops *q_ops,
928 				struct mlxsw_pci_queue *q, u8 q_num)
929 {
930 	struct mlxsw_pci_mem_item *mem_item = &q->mem_item;
931 	int i;
932 	int err;
933 
934 	spin_lock_init(&q->lock);
935 	q->num = q_num;
936 	q->count = q_ops->elem_count;
937 	q->elem_size = q_ops->elem_size;
938 	q->type = q_ops->type;
939 	q->pci = mlxsw_pci;
940 
941 	if (q_ops->tasklet)
942 		tasklet_init(&q->tasklet, q_ops->tasklet, (unsigned long) q);
943 
944 	mem_item->size = MLXSW_PCI_AQ_SIZE;
945 	mem_item->buf = pci_alloc_consistent(mlxsw_pci->pdev,
946 					     mem_item->size,
947 					     &mem_item->mapaddr);
948 	if (!mem_item->buf)
949 		return -ENOMEM;
950 	memset(mem_item->buf, 0, mem_item->size);
951 
952 	q->elem_info = kcalloc(q->count, sizeof(*q->elem_info), GFP_KERNEL);
953 	if (!q->elem_info) {
954 		err = -ENOMEM;
955 		goto err_elem_info_alloc;
956 	}
957 
958 	/* Initialize dma mapped elements info elem_info for
959 	 * future easy access.
960 	 */
961 	for (i = 0; i < q->count; i++) {
962 		struct mlxsw_pci_queue_elem_info *elem_info;
963 
964 		elem_info = mlxsw_pci_queue_elem_info_get(q, i);
965 		elem_info->elem =
966 			__mlxsw_pci_queue_elem_get(q, q_ops->elem_size, i);
967 	}
968 
969 	mlxsw_cmd_mbox_zero(mbox);
970 	err = q_ops->init(mlxsw_pci, mbox, q);
971 	if (err)
972 		goto err_q_ops_init;
973 	return 0;
974 
975 err_q_ops_init:
976 	kfree(q->elem_info);
977 err_elem_info_alloc:
978 	pci_free_consistent(mlxsw_pci->pdev, mem_item->size,
979 			    mem_item->buf, mem_item->mapaddr);
980 	return err;
981 }
982 
983 static void mlxsw_pci_queue_fini(struct mlxsw_pci *mlxsw_pci,
984 				 const struct mlxsw_pci_queue_ops *q_ops,
985 				 struct mlxsw_pci_queue *q)
986 {
987 	struct mlxsw_pci_mem_item *mem_item = &q->mem_item;
988 
989 	q_ops->fini(mlxsw_pci, q);
990 	kfree(q->elem_info);
991 	pci_free_consistent(mlxsw_pci->pdev, mem_item->size,
992 			    mem_item->buf, mem_item->mapaddr);
993 }
994 
995 static int mlxsw_pci_queue_group_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
996 				      const struct mlxsw_pci_queue_ops *q_ops,
997 				      u8 num_qs)
998 {
999 	struct pci_dev *pdev = mlxsw_pci->pdev;
1000 	struct mlxsw_pci_queue_type_group *queue_group;
1001 	char tmp[16];
1002 	int i;
1003 	int err;
1004 
1005 	queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);
1006 	queue_group->q = kcalloc(num_qs, sizeof(*queue_group->q), GFP_KERNEL);
1007 	if (!queue_group->q)
1008 		return -ENOMEM;
1009 
1010 	for (i = 0; i < num_qs; i++) {
1011 		err = mlxsw_pci_queue_init(mlxsw_pci, mbox, q_ops,
1012 					   &queue_group->q[i], i);
1013 		if (err)
1014 			goto err_queue_init;
1015 	}
1016 	queue_group->count = num_qs;
1017 
1018 	sprintf(tmp, "%s_stats", mlxsw_pci_queue_type_str(q_ops->type));
1019 	debugfs_create_devm_seqfile(&pdev->dev, tmp, mlxsw_pci->dbg_dir,
1020 				    q_ops->dbg_read);
1021 
1022 	return 0;
1023 
1024 err_queue_init:
1025 	for (i--; i >= 0; i--)
1026 		mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);
1027 	kfree(queue_group->q);
1028 	return err;
1029 }
1030 
1031 static void mlxsw_pci_queue_group_fini(struct mlxsw_pci *mlxsw_pci,
1032 				       const struct mlxsw_pci_queue_ops *q_ops)
1033 {
1034 	struct mlxsw_pci_queue_type_group *queue_group;
1035 	int i;
1036 
1037 	queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_ops->type);
1038 	for (i = 0; i < queue_group->count; i++)
1039 		mlxsw_pci_queue_fini(mlxsw_pci, q_ops, &queue_group->q[i]);
1040 	kfree(queue_group->q);
1041 }
1042 
1043 static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox)
1044 {
1045 	struct pci_dev *pdev = mlxsw_pci->pdev;
1046 	u8 num_sdqs;
1047 	u8 sdq_log2sz;
1048 	u8 num_rdqs;
1049 	u8 rdq_log2sz;
1050 	u8 num_cqs;
1051 	u8 cq_log2sz;
1052 	u8 num_eqs;
1053 	u8 eq_log2sz;
1054 	int err;
1055 
1056 	mlxsw_cmd_mbox_zero(mbox);
1057 	err = mlxsw_cmd_query_aq_cap(mlxsw_pci->core, mbox);
1058 	if (err)
1059 		return err;
1060 
1061 	num_sdqs = mlxsw_cmd_mbox_query_aq_cap_max_num_sdqs_get(mbox);
1062 	sdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_sdq_sz_get(mbox);
1063 	num_rdqs = mlxsw_cmd_mbox_query_aq_cap_max_num_rdqs_get(mbox);
1064 	rdq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_rdq_sz_get(mbox);
1065 	num_cqs = mlxsw_cmd_mbox_query_aq_cap_max_num_cqs_get(mbox);
1066 	cq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_cq_sz_get(mbox);
1067 	num_eqs = mlxsw_cmd_mbox_query_aq_cap_max_num_eqs_get(mbox);
1068 	eq_log2sz = mlxsw_cmd_mbox_query_aq_cap_log_max_eq_sz_get(mbox);
1069 
1070 	if ((num_sdqs != MLXSW_PCI_SDQS_COUNT) ||
1071 	    (num_rdqs != MLXSW_PCI_RDQS_COUNT) ||
1072 	    (num_cqs != MLXSW_PCI_CQS_COUNT) ||
1073 	    (num_eqs != MLXSW_PCI_EQS_COUNT)) {
1074 		dev_err(&pdev->dev, "Unsupported number of queues\n");
1075 		return -EINVAL;
1076 	}
1077 
1078 	if ((1 << sdq_log2sz != MLXSW_PCI_WQE_COUNT) ||
1079 	    (1 << rdq_log2sz != MLXSW_PCI_WQE_COUNT) ||
1080 	    (1 << cq_log2sz != MLXSW_PCI_CQE_COUNT) ||
1081 	    (1 << eq_log2sz != MLXSW_PCI_EQE_COUNT)) {
1082 		dev_err(&pdev->dev, "Unsupported number of async queue descriptors\n");
1083 		return -EINVAL;
1084 	}
1085 
1086 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops,
1087 					 num_eqs);
1088 	if (err) {
1089 		dev_err(&pdev->dev, "Failed to initialize event queues\n");
1090 		return err;
1091 	}
1092 
1093 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_cq_ops,
1094 					 num_cqs);
1095 	if (err) {
1096 		dev_err(&pdev->dev, "Failed to initialize completion queues\n");
1097 		goto err_cqs_init;
1098 	}
1099 
1100 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_sdq_ops,
1101 					 num_sdqs);
1102 	if (err) {
1103 		dev_err(&pdev->dev, "Failed to initialize send descriptor queues\n");
1104 		goto err_sdqs_init;
1105 	}
1106 
1107 	err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_rdq_ops,
1108 					 num_rdqs);
1109 	if (err) {
1110 		dev_err(&pdev->dev, "Failed to initialize receive descriptor queues\n");
1111 		goto err_rdqs_init;
1112 	}
1113 
1114 	/* We have to poll in command interface until queues are initialized */
1115 	mlxsw_pci->cmd.nopoll = true;
1116 	return 0;
1117 
1118 err_rdqs_init:
1119 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops);
1120 err_sdqs_init:
1121 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops);
1122 err_cqs_init:
1123 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_eq_ops);
1124 	return err;
1125 }
1126 
1127 static void mlxsw_pci_aqs_fini(struct mlxsw_pci *mlxsw_pci)
1128 {
1129 	mlxsw_pci->cmd.nopoll = false;
1130 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_rdq_ops);
1131 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops);
1132 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops);
1133 	mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_eq_ops);
1134 }
1135 
1136 static void
1137 mlxsw_pci_config_profile_swid_config(struct mlxsw_pci *mlxsw_pci,
1138 				     char *mbox, int index,
1139 				     const struct mlxsw_swid_config *swid)
1140 {
1141 	u8 mask = 0;
1142 
1143 	if (swid->used_type) {
1144 		mlxsw_cmd_mbox_config_profile_swid_config_type_set(
1145 			mbox, index, swid->type);
1146 		mask |= 1;
1147 	}
1148 	if (swid->used_properties) {
1149 		mlxsw_cmd_mbox_config_profile_swid_config_properties_set(
1150 			mbox, index, swid->properties);
1151 		mask |= 2;
1152 	}
1153 	mlxsw_cmd_mbox_config_profile_swid_config_mask_set(mbox, index, mask);
1154 }
1155 
1156 static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
1157 				    const struct mlxsw_config_profile *profile)
1158 {
1159 	int i;
1160 
1161 	mlxsw_cmd_mbox_zero(mbox);
1162 
1163 	if (profile->used_max_vepa_channels) {
1164 		mlxsw_cmd_mbox_config_profile_set_max_vepa_channels_set(
1165 			mbox, 1);
1166 		mlxsw_cmd_mbox_config_profile_max_vepa_channels_set(
1167 			mbox, profile->max_vepa_channels);
1168 	}
1169 	if (profile->used_max_lag) {
1170 		mlxsw_cmd_mbox_config_profile_set_max_lag_set(
1171 			mbox, 1);
1172 		mlxsw_cmd_mbox_config_profile_max_lag_set(
1173 			mbox, profile->max_lag);
1174 	}
1175 	if (profile->used_max_port_per_lag) {
1176 		mlxsw_cmd_mbox_config_profile_set_max_port_per_lag_set(
1177 			mbox, 1);
1178 		mlxsw_cmd_mbox_config_profile_max_port_per_lag_set(
1179 			mbox, profile->max_port_per_lag);
1180 	}
1181 	if (profile->used_max_mid) {
1182 		mlxsw_cmd_mbox_config_profile_set_max_mid_set(
1183 			mbox, 1);
1184 		mlxsw_cmd_mbox_config_profile_max_mid_set(
1185 			mbox, profile->max_mid);
1186 	}
1187 	if (profile->used_max_pgt) {
1188 		mlxsw_cmd_mbox_config_profile_set_max_pgt_set(
1189 			mbox, 1);
1190 		mlxsw_cmd_mbox_config_profile_max_pgt_set(
1191 			mbox, profile->max_pgt);
1192 	}
1193 	if (profile->used_max_system_port) {
1194 		mlxsw_cmd_mbox_config_profile_set_max_system_port_set(
1195 			mbox, 1);
1196 		mlxsw_cmd_mbox_config_profile_max_system_port_set(
1197 			mbox, profile->max_system_port);
1198 	}
1199 	if (profile->used_max_vlan_groups) {
1200 		mlxsw_cmd_mbox_config_profile_set_max_vlan_groups_set(
1201 			mbox, 1);
1202 		mlxsw_cmd_mbox_config_profile_max_vlan_groups_set(
1203 			mbox, profile->max_vlan_groups);
1204 	}
1205 	if (profile->used_max_regions) {
1206 		mlxsw_cmd_mbox_config_profile_set_max_regions_set(
1207 			mbox, 1);
1208 		mlxsw_cmd_mbox_config_profile_max_regions_set(
1209 			mbox, profile->max_regions);
1210 	}
1211 	if (profile->used_flood_tables) {
1212 		mlxsw_cmd_mbox_config_profile_set_flood_tables_set(
1213 			mbox, 1);
1214 		mlxsw_cmd_mbox_config_profile_max_flood_tables_set(
1215 			mbox, profile->max_flood_tables);
1216 		mlxsw_cmd_mbox_config_profile_max_vid_flood_tables_set(
1217 			mbox, profile->max_vid_flood_tables);
1218 	}
1219 	if (profile->used_flood_mode) {
1220 		mlxsw_cmd_mbox_config_profile_set_flood_mode_set(
1221 			mbox, 1);
1222 		mlxsw_cmd_mbox_config_profile_flood_mode_set(
1223 			mbox, profile->flood_mode);
1224 	}
1225 	if (profile->used_max_ib_mc) {
1226 		mlxsw_cmd_mbox_config_profile_set_max_ib_mc_set(
1227 			mbox, 1);
1228 		mlxsw_cmd_mbox_config_profile_max_ib_mc_set(
1229 			mbox, profile->max_ib_mc);
1230 	}
1231 	if (profile->used_max_pkey) {
1232 		mlxsw_cmd_mbox_config_profile_set_max_pkey_set(
1233 			mbox, 1);
1234 		mlxsw_cmd_mbox_config_profile_max_pkey_set(
1235 			mbox, profile->max_pkey);
1236 	}
1237 	if (profile->used_ar_sec) {
1238 		mlxsw_cmd_mbox_config_profile_set_ar_sec_set(
1239 			mbox, 1);
1240 		mlxsw_cmd_mbox_config_profile_ar_sec_set(
1241 			mbox, profile->ar_sec);
1242 	}
1243 	if (profile->used_adaptive_routing_group_cap) {
1244 		mlxsw_cmd_mbox_config_profile_set_adaptive_routing_group_cap_set(
1245 			mbox, 1);
1246 		mlxsw_cmd_mbox_config_profile_adaptive_routing_group_cap_set(
1247 			mbox, profile->adaptive_routing_group_cap);
1248 	}
1249 
1250 	for (i = 0; i < MLXSW_CONFIG_PROFILE_SWID_COUNT; i++)
1251 		mlxsw_pci_config_profile_swid_config(mlxsw_pci, mbox, i,
1252 						     &profile->swid_config[i]);
1253 
1254 	return mlxsw_cmd_config_profile_set(mlxsw_pci->core, mbox);
1255 }
1256 
1257 static int mlxsw_pci_boardinfo(struct mlxsw_pci *mlxsw_pci, char *mbox)
1258 {
1259 	struct mlxsw_bus_info *bus_info = &mlxsw_pci->bus_info;
1260 	int err;
1261 
1262 	mlxsw_cmd_mbox_zero(mbox);
1263 	err = mlxsw_cmd_boardinfo(mlxsw_pci->core, mbox);
1264 	if (err)
1265 		return err;
1266 	mlxsw_cmd_mbox_boardinfo_vsd_memcpy_from(mbox, bus_info->vsd);
1267 	mlxsw_cmd_mbox_boardinfo_psid_memcpy_from(mbox, bus_info->psid);
1268 	return 0;
1269 }
1270 
1271 static int mlxsw_pci_fw_area_init(struct mlxsw_pci *mlxsw_pci, char *mbox,
1272 				  u16 num_pages)
1273 {
1274 	struct mlxsw_pci_mem_item *mem_item;
1275 	int i;
1276 	int err;
1277 
1278 	mlxsw_pci->fw_area.items = kcalloc(num_pages, sizeof(*mem_item),
1279 					   GFP_KERNEL);
1280 	if (!mlxsw_pci->fw_area.items)
1281 		return -ENOMEM;
1282 	mlxsw_pci->fw_area.num_pages = num_pages;
1283 
1284 	mlxsw_cmd_mbox_zero(mbox);
1285 	for (i = 0; i < num_pages; i++) {
1286 		mem_item = &mlxsw_pci->fw_area.items[i];
1287 
1288 		mem_item->size = MLXSW_PCI_PAGE_SIZE;
1289 		mem_item->buf = pci_alloc_consistent(mlxsw_pci->pdev,
1290 						     mem_item->size,
1291 						     &mem_item->mapaddr);
1292 		if (!mem_item->buf) {
1293 			err = -ENOMEM;
1294 			goto err_alloc;
1295 		}
1296 		mlxsw_cmd_mbox_map_fa_pa_set(mbox, i, mem_item->mapaddr);
1297 		mlxsw_cmd_mbox_map_fa_log2size_set(mbox, i, 0); /* 1 page */
1298 	}
1299 
1300 	err = mlxsw_cmd_map_fa(mlxsw_pci->core, mbox, num_pages);
1301 	if (err)
1302 		goto err_cmd_map_fa;
1303 
1304 	return 0;
1305 
1306 err_cmd_map_fa:
1307 err_alloc:
1308 	for (i--; i >= 0; i--) {
1309 		mem_item = &mlxsw_pci->fw_area.items[i];
1310 
1311 		pci_free_consistent(mlxsw_pci->pdev, mem_item->size,
1312 				    mem_item->buf, mem_item->mapaddr);
1313 	}
1314 	kfree(mlxsw_pci->fw_area.items);
1315 	return err;
1316 }
1317 
1318 static void mlxsw_pci_fw_area_fini(struct mlxsw_pci *mlxsw_pci)
1319 {
1320 	struct mlxsw_pci_mem_item *mem_item;
1321 	int i;
1322 
1323 	mlxsw_cmd_unmap_fa(mlxsw_pci->core);
1324 
1325 	for (i = 0; i < mlxsw_pci->fw_area.num_pages; i++) {
1326 		mem_item = &mlxsw_pci->fw_area.items[i];
1327 
1328 		pci_free_consistent(mlxsw_pci->pdev, mem_item->size,
1329 				    mem_item->buf, mem_item->mapaddr);
1330 	}
1331 	kfree(mlxsw_pci->fw_area.items);
1332 }
1333 
1334 static irqreturn_t mlxsw_pci_eq_irq_handler(int irq, void *dev_id)
1335 {
1336 	struct mlxsw_pci *mlxsw_pci = dev_id;
1337 	struct mlxsw_pci_queue *q;
1338 	int i;
1339 
1340 	for (i = 0; i < MLXSW_PCI_EQS_COUNT; i++) {
1341 		q = mlxsw_pci_eq_get(mlxsw_pci, i);
1342 		mlxsw_pci_queue_tasklet_schedule(q);
1343 	}
1344 	return IRQ_HANDLED;
1345 }
1346 
1347 static int mlxsw_pci_mbox_alloc(struct mlxsw_pci *mlxsw_pci,
1348 				struct mlxsw_pci_mem_item *mbox)
1349 {
1350 	struct pci_dev *pdev = mlxsw_pci->pdev;
1351 	int err = 0;
1352 
1353 	mbox->size = MLXSW_CMD_MBOX_SIZE;
1354 	mbox->buf = pci_alloc_consistent(pdev, MLXSW_CMD_MBOX_SIZE,
1355 					 &mbox->mapaddr);
1356 	if (!mbox->buf) {
1357 		dev_err(&pdev->dev, "Failed allocating memory for mailbox\n");
1358 		err = -ENOMEM;
1359 	}
1360 
1361 	return err;
1362 }
1363 
1364 static void mlxsw_pci_mbox_free(struct mlxsw_pci *mlxsw_pci,
1365 				struct mlxsw_pci_mem_item *mbox)
1366 {
1367 	struct pci_dev *pdev = mlxsw_pci->pdev;
1368 
1369 	pci_free_consistent(pdev, MLXSW_CMD_MBOX_SIZE, mbox->buf,
1370 			    mbox->mapaddr);
1371 }
1372 
1373 static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
1374 			  const struct mlxsw_config_profile *profile)
1375 {
1376 	struct mlxsw_pci *mlxsw_pci = bus_priv;
1377 	struct pci_dev *pdev = mlxsw_pci->pdev;
1378 	char *mbox;
1379 	u16 num_pages;
1380 	int err;
1381 
1382 	mutex_init(&mlxsw_pci->cmd.lock);
1383 	init_waitqueue_head(&mlxsw_pci->cmd.wait);
1384 
1385 	mlxsw_pci->core = mlxsw_core;
1386 
1387 	mbox = mlxsw_cmd_mbox_alloc();
1388 	if (!mbox)
1389 		return -ENOMEM;
1390 
1391 	err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
1392 	if (err)
1393 		goto mbox_put;
1394 
1395 	err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
1396 	if (err)
1397 		goto err_out_mbox_alloc;
1398 
1399 	err = mlxsw_cmd_query_fw(mlxsw_core, mbox);
1400 	if (err)
1401 		goto err_query_fw;
1402 
1403 	mlxsw_pci->bus_info.fw_rev.major =
1404 		mlxsw_cmd_mbox_query_fw_fw_rev_major_get(mbox);
1405 	mlxsw_pci->bus_info.fw_rev.minor =
1406 		mlxsw_cmd_mbox_query_fw_fw_rev_minor_get(mbox);
1407 	mlxsw_pci->bus_info.fw_rev.subminor =
1408 		mlxsw_cmd_mbox_query_fw_fw_rev_subminor_get(mbox);
1409 
1410 	if (mlxsw_cmd_mbox_query_fw_cmd_interface_rev_get(mbox) != 1) {
1411 		dev_err(&pdev->dev, "Unsupported cmd interface revision ID queried from hw\n");
1412 		err = -EINVAL;
1413 		goto err_iface_rev;
1414 	}
1415 	if (mlxsw_cmd_mbox_query_fw_doorbell_page_bar_get(mbox) != 0) {
1416 		dev_err(&pdev->dev, "Unsupported doorbell page bar queried from hw\n");
1417 		err = -EINVAL;
1418 		goto err_doorbell_page_bar;
1419 	}
1420 
1421 	mlxsw_pci->doorbell_offset =
1422 		mlxsw_cmd_mbox_query_fw_doorbell_page_offset_get(mbox);
1423 
1424 	num_pages = mlxsw_cmd_mbox_query_fw_fw_pages_get(mbox);
1425 	err = mlxsw_pci_fw_area_init(mlxsw_pci, mbox, num_pages);
1426 	if (err)
1427 		goto err_fw_area_init;
1428 
1429 	err = mlxsw_pci_boardinfo(mlxsw_pci, mbox);
1430 	if (err)
1431 		goto err_boardinfo;
1432 
1433 	err = mlxsw_pci_config_profile(mlxsw_pci, mbox, profile);
1434 	if (err)
1435 		goto err_config_profile;
1436 
1437 	err = mlxsw_pci_aqs_init(mlxsw_pci, mbox);
1438 	if (err)
1439 		goto err_aqs_init;
1440 
1441 	err = request_irq(mlxsw_pci->msix_entry.vector,
1442 			  mlxsw_pci_eq_irq_handler, 0,
1443 			  mlxsw_pci_driver_name, mlxsw_pci);
1444 	if (err) {
1445 		dev_err(&pdev->dev, "IRQ request failed\n");
1446 		goto err_request_eq_irq;
1447 	}
1448 
1449 	goto mbox_put;
1450 
1451 err_request_eq_irq:
1452 	mlxsw_pci_aqs_fini(mlxsw_pci);
1453 err_aqs_init:
1454 err_config_profile:
1455 err_boardinfo:
1456 	mlxsw_pci_fw_area_fini(mlxsw_pci);
1457 err_fw_area_init:
1458 err_doorbell_page_bar:
1459 err_iface_rev:
1460 err_query_fw:
1461 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
1462 err_out_mbox_alloc:
1463 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
1464 mbox_put:
1465 	mlxsw_cmd_mbox_free(mbox);
1466 	return err;
1467 }
1468 
1469 static void mlxsw_pci_fini(void *bus_priv)
1470 {
1471 	struct mlxsw_pci *mlxsw_pci = bus_priv;
1472 
1473 	free_irq(mlxsw_pci->msix_entry.vector, mlxsw_pci);
1474 	mlxsw_pci_aqs_fini(mlxsw_pci);
1475 	mlxsw_pci_fw_area_fini(mlxsw_pci);
1476 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.out_mbox);
1477 	mlxsw_pci_mbox_free(mlxsw_pci, &mlxsw_pci->cmd.in_mbox);
1478 }
1479 
1480 static struct mlxsw_pci_queue *
1481 mlxsw_pci_sdq_pick(struct mlxsw_pci *mlxsw_pci,
1482 		   const struct mlxsw_tx_info *tx_info)
1483 {
1484 	u8 sdqn = tx_info->local_port % mlxsw_pci_sdq_count(mlxsw_pci);
1485 
1486 	return mlxsw_pci_sdq_get(mlxsw_pci, sdqn);
1487 }
1488 
1489 static bool mlxsw_pci_skb_transmit_busy(void *bus_priv,
1490 					const struct mlxsw_tx_info *tx_info)
1491 {
1492 	struct mlxsw_pci *mlxsw_pci = bus_priv;
1493 	struct mlxsw_pci_queue *q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);
1494 
1495 	return !mlxsw_pci_queue_elem_info_producer_get(q);
1496 }
1497 
1498 static int mlxsw_pci_skb_transmit(void *bus_priv, struct sk_buff *skb,
1499 				  const struct mlxsw_tx_info *tx_info)
1500 {
1501 	struct mlxsw_pci *mlxsw_pci = bus_priv;
1502 	struct mlxsw_pci_queue *q;
1503 	struct mlxsw_pci_queue_elem_info *elem_info;
1504 	char *wqe;
1505 	int i;
1506 	int err;
1507 
1508 	if (skb_shinfo(skb)->nr_frags > MLXSW_PCI_WQE_SG_ENTRIES - 1) {
1509 		err = skb_linearize(skb);
1510 		if (err)
1511 			return err;
1512 	}
1513 
1514 	q = mlxsw_pci_sdq_pick(mlxsw_pci, tx_info);
1515 	spin_lock_bh(&q->lock);
1516 	elem_info = mlxsw_pci_queue_elem_info_producer_get(q);
1517 	if (!elem_info) {
1518 		/* queue is full */
1519 		err = -EAGAIN;
1520 		goto unlock;
1521 	}
1522 	elem_info->u.sdq.skb = skb;
1523 
1524 	wqe = elem_info->elem;
1525 	mlxsw_pci_wqe_c_set(wqe, 1); /* always report completion */
1526 	mlxsw_pci_wqe_lp_set(wqe, !!tx_info->is_emad);
1527 	mlxsw_pci_wqe_type_set(wqe, MLXSW_PCI_WQE_TYPE_ETHERNET);
1528 
1529 	err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, 0, skb->data,
1530 				     skb_headlen(skb), DMA_TO_DEVICE);
1531 	if (err)
1532 		goto unlock;
1533 
1534 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1535 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1536 
1537 		err = mlxsw_pci_wqe_frag_map(mlxsw_pci, wqe, i + 1,
1538 					     skb_frag_address(frag),
1539 					     skb_frag_size(frag),
1540 					     DMA_TO_DEVICE);
1541 		if (err)
1542 			goto unmap_frags;
1543 	}
1544 
1545 	/* Set unused sq entries byte count to zero. */
1546 	for (i++; i < MLXSW_PCI_WQE_SG_ENTRIES; i++)
1547 		mlxsw_pci_wqe_byte_count_set(wqe, i, 0);
1548 
1549 	/* Everything is set up, ring producer doorbell to get HW going */
1550 	q->producer_counter++;
1551 	mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q);
1552 
1553 	goto unlock;
1554 
1555 unmap_frags:
1556 	for (; i >= 0; i--)
1557 		mlxsw_pci_wqe_frag_unmap(mlxsw_pci, wqe, i, DMA_TO_DEVICE);
1558 unlock:
1559 	spin_unlock_bh(&q->lock);
1560 	return err;
1561 }
1562 
1563 static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod,
1564 			      u32 in_mod, bool out_mbox_direct,
1565 			      char *in_mbox, size_t in_mbox_size,
1566 			      char *out_mbox, size_t out_mbox_size,
1567 			      u8 *p_status)
1568 {
1569 	struct mlxsw_pci *mlxsw_pci = bus_priv;
1570 	dma_addr_t in_mapaddr = mlxsw_pci->cmd.in_mbox.mapaddr;
1571 	dma_addr_t out_mapaddr = mlxsw_pci->cmd.out_mbox.mapaddr;
1572 	bool evreq = mlxsw_pci->cmd.nopoll;
1573 	unsigned long timeout = msecs_to_jiffies(MLXSW_PCI_CIR_TIMEOUT_MSECS);
1574 	bool *p_wait_done = &mlxsw_pci->cmd.wait_done;
1575 	int err;
1576 
1577 	*p_status = MLXSW_CMD_STATUS_OK;
1578 
1579 	err = mutex_lock_interruptible(&mlxsw_pci->cmd.lock);
1580 	if (err)
1581 		return err;
1582 
1583 	if (in_mbox)
1584 		memcpy(mlxsw_pci->cmd.in_mbox.buf, in_mbox, in_mbox_size);
1585 	mlxsw_pci_write32(mlxsw_pci, CIR_IN_PARAM_HI, in_mapaddr >> 32);
1586 	mlxsw_pci_write32(mlxsw_pci, CIR_IN_PARAM_LO, in_mapaddr);
1587 
1588 	mlxsw_pci_write32(mlxsw_pci, CIR_OUT_PARAM_HI, out_mapaddr >> 32);
1589 	mlxsw_pci_write32(mlxsw_pci, CIR_OUT_PARAM_LO, out_mapaddr);
1590 
1591 	mlxsw_pci_write32(mlxsw_pci, CIR_IN_MODIFIER, in_mod);
1592 	mlxsw_pci_write32(mlxsw_pci, CIR_TOKEN, 0);
1593 
1594 	*p_wait_done = false;
1595 
1596 	wmb(); /* all needs to be written before we write control register */
1597 	mlxsw_pci_write32(mlxsw_pci, CIR_CTRL,
1598 			  MLXSW_PCI_CIR_CTRL_GO_BIT |
1599 			  (evreq ? MLXSW_PCI_CIR_CTRL_EVREQ_BIT : 0) |
1600 			  (opcode_mod << MLXSW_PCI_CIR_CTRL_OPCODE_MOD_SHIFT) |
1601 			  opcode);
1602 
1603 	if (!evreq) {
1604 		unsigned long end;
1605 
1606 		end = jiffies + timeout;
1607 		do {
1608 			u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL);
1609 
1610 			if (!(ctrl & MLXSW_PCI_CIR_CTRL_GO_BIT)) {
1611 				*p_wait_done = true;
1612 				*p_status = ctrl >> MLXSW_PCI_CIR_CTRL_STATUS_SHIFT;
1613 				break;
1614 			}
1615 			cond_resched();
1616 		} while (time_before(jiffies, end));
1617 	} else {
1618 		wait_event_timeout(mlxsw_pci->cmd.wait, *p_wait_done, timeout);
1619 		*p_status = mlxsw_pci->cmd.comp.status;
1620 	}
1621 
1622 	err = 0;
1623 	if (*p_wait_done) {
1624 		if (*p_status)
1625 			err = -EIO;
1626 	} else {
1627 		err = -ETIMEDOUT;
1628 	}
1629 
1630 	if (!err && out_mbox && out_mbox_direct) {
1631 		/* Some commands don't use output param as address to mailbox
1632 		 * but they store output directly into registers. In that case,
1633 		 * copy registers into mbox buffer.
1634 		 */
1635 		__be32 tmp;
1636 
1637 		if (!evreq) {
1638 			tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci,
1639 							   CIR_OUT_PARAM_HI));
1640 			memcpy(out_mbox, &tmp, sizeof(tmp));
1641 			tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci,
1642 							   CIR_OUT_PARAM_LO));
1643 			memcpy(out_mbox + sizeof(tmp), &tmp, sizeof(tmp));
1644 		}
1645 	} else if (!err && out_mbox)
1646 		memcpy(out_mbox, mlxsw_pci->cmd.out_mbox.buf, out_mbox_size);
1647 
1648 	mutex_unlock(&mlxsw_pci->cmd.lock);
1649 
1650 	return err;
1651 }
1652 
1653 static const struct mlxsw_bus mlxsw_pci_bus = {
1654 	.kind			= "pci",
1655 	.init			= mlxsw_pci_init,
1656 	.fini			= mlxsw_pci_fini,
1657 	.skb_transmit_busy	= mlxsw_pci_skb_transmit_busy,
1658 	.skb_transmit		= mlxsw_pci_skb_transmit,
1659 	.cmd_exec		= mlxsw_pci_cmd_exec,
1660 };
1661 
1662 static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci)
1663 {
1664 	mlxsw_pci_write32(mlxsw_pci, SW_RESET, MLXSW_PCI_SW_RESET_RST_BIT);
1665 	/* Current firware does not let us know when the reset is done.
1666 	 * So we just wait here for constant time and hope for the best.
1667 	 */
1668 	msleep(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS);
1669 	return 0;
1670 }
1671 
1672 static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1673 {
1674 	struct mlxsw_pci *mlxsw_pci;
1675 	int err;
1676 
1677 	mlxsw_pci = kzalloc(sizeof(*mlxsw_pci), GFP_KERNEL);
1678 	if (!mlxsw_pci)
1679 		return -ENOMEM;
1680 
1681 	err = pci_enable_device(pdev);
1682 	if (err) {
1683 		dev_err(&pdev->dev, "pci_enable_device failed\n");
1684 		goto err_pci_enable_device;
1685 	}
1686 
1687 	err = pci_request_regions(pdev, mlxsw_pci_driver_name);
1688 	if (err) {
1689 		dev_err(&pdev->dev, "pci_request_regions failed\n");
1690 		goto err_pci_request_regions;
1691 	}
1692 
1693 	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1694 	if (!err) {
1695 		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1696 		if (err) {
1697 			dev_err(&pdev->dev, "pci_set_consistent_dma_mask failed\n");
1698 			goto err_pci_set_dma_mask;
1699 		}
1700 	} else {
1701 		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1702 		if (err) {
1703 			dev_err(&pdev->dev, "pci_set_dma_mask failed\n");
1704 			goto err_pci_set_dma_mask;
1705 		}
1706 	}
1707 
1708 	if (pci_resource_len(pdev, 0) < MLXSW_PCI_BAR0_SIZE) {
1709 		dev_err(&pdev->dev, "invalid PCI region size\n");
1710 		err = -EINVAL;
1711 		goto err_pci_resource_len_check;
1712 	}
1713 
1714 	mlxsw_pci->hw_addr = ioremap(pci_resource_start(pdev, 0),
1715 				     pci_resource_len(pdev, 0));
1716 	if (!mlxsw_pci->hw_addr) {
1717 		dev_err(&pdev->dev, "ioremap failed\n");
1718 		err = -EIO;
1719 		goto err_ioremap;
1720 	}
1721 	pci_set_master(pdev);
1722 
1723 	mlxsw_pci->pdev = pdev;
1724 	pci_set_drvdata(pdev, mlxsw_pci);
1725 
1726 	err = mlxsw_pci_sw_reset(mlxsw_pci);
1727 	if (err) {
1728 		dev_err(&pdev->dev, "Software reset failed\n");
1729 		goto err_sw_reset;
1730 	}
1731 
1732 	err = pci_enable_msix_exact(pdev, &mlxsw_pci->msix_entry, 1);
1733 	if (err) {
1734 		dev_err(&pdev->dev, "MSI-X init failed\n");
1735 		goto err_msix_init;
1736 	}
1737 
1738 	mlxsw_pci->bus_info.device_kind = mlxsw_pci_device_kind_get(id);
1739 	mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev);
1740 	mlxsw_pci->bus_info.dev = &pdev->dev;
1741 
1742 	mlxsw_pci->dbg_dir = debugfs_create_dir(mlxsw_pci->bus_info.device_name,
1743 						mlxsw_pci_dbg_root);
1744 	if (!mlxsw_pci->dbg_dir) {
1745 		dev_err(&pdev->dev, "Failed to create debugfs dir\n");
1746 		err = -ENOMEM;
1747 		goto err_dbg_create_dir;
1748 	}
1749 
1750 	err = mlxsw_core_bus_device_register(&mlxsw_pci->bus_info,
1751 					     &mlxsw_pci_bus, mlxsw_pci);
1752 	if (err) {
1753 		dev_err(&pdev->dev, "cannot register bus device\n");
1754 		goto err_bus_device_register;
1755 	}
1756 
1757 	return 0;
1758 
1759 err_bus_device_register:
1760 	debugfs_remove_recursive(mlxsw_pci->dbg_dir);
1761 err_dbg_create_dir:
1762 	pci_disable_msix(mlxsw_pci->pdev);
1763 err_msix_init:
1764 err_sw_reset:
1765 	iounmap(mlxsw_pci->hw_addr);
1766 err_ioremap:
1767 err_pci_resource_len_check:
1768 err_pci_set_dma_mask:
1769 	pci_release_regions(pdev);
1770 err_pci_request_regions:
1771 	pci_disable_device(pdev);
1772 err_pci_enable_device:
1773 	kfree(mlxsw_pci);
1774 	return err;
1775 }
1776 
1777 static void mlxsw_pci_remove(struct pci_dev *pdev)
1778 {
1779 	struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev);
1780 
1781 	mlxsw_core_bus_device_unregister(mlxsw_pci->core);
1782 	debugfs_remove_recursive(mlxsw_pci->dbg_dir);
1783 	pci_disable_msix(mlxsw_pci->pdev);
1784 	iounmap(mlxsw_pci->hw_addr);
1785 	pci_release_regions(mlxsw_pci->pdev);
1786 	pci_disable_device(mlxsw_pci->pdev);
1787 	kfree(mlxsw_pci);
1788 }
1789 
1790 static struct pci_driver mlxsw_pci_driver = {
1791 	.name		= mlxsw_pci_driver_name,
1792 	.id_table	= mlxsw_pci_id_table,
1793 	.probe		= mlxsw_pci_probe,
1794 	.remove		= mlxsw_pci_remove,
1795 };
1796 
1797 static int __init mlxsw_pci_module_init(void)
1798 {
1799 	int err;
1800 
1801 	mlxsw_pci_dbg_root = debugfs_create_dir(mlxsw_pci_driver_name, NULL);
1802 	if (!mlxsw_pci_dbg_root)
1803 		return -ENOMEM;
1804 	err = pci_register_driver(&mlxsw_pci_driver);
1805 	if (err)
1806 		goto err_register_driver;
1807 	return 0;
1808 
1809 err_register_driver:
1810 	debugfs_remove_recursive(mlxsw_pci_dbg_root);
1811 	return err;
1812 }
1813 
1814 static void __exit mlxsw_pci_module_exit(void)
1815 {
1816 	pci_unregister_driver(&mlxsw_pci_driver);
1817 	debugfs_remove_recursive(mlxsw_pci_dbg_root);
1818 }
1819 
1820 module_init(mlxsw_pci_module_init);
1821 module_exit(mlxsw_pci_module_exit);
1822 
1823 MODULE_LICENSE("Dual BSD/GPL");
1824 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
1825 MODULE_DESCRIPTION("Mellanox switch PCI interface driver");
1826 MODULE_DEVICE_TABLE(pci, mlxsw_pci_id_table);
1827