1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/device.h>
7 #include <linux/export.h>
8 #include <linux/err.h>
9 #include <linux/if_link.h>
10 #include <linux/netdevice.h>
11 #include <linux/completion.h>
12 #include <linux/skbuff.h>
13 #include <linux/etherdevice.h>
14 #include <linux/types.h>
15 #include <linux/string.h>
16 #include <linux/gfp.h>
17 #include <linux/random.h>
18 #include <linux/jiffies.h>
19 #include <linux/mutex.h>
20 #include <linux/rcupdate.h>
21 #include <linux/slab.h>
22 #include <linux/workqueue.h>
23 #include <asm/byteorder.h>
24 #include <net/devlink.h>
25 #include <trace/events/devlink.h>
26 
27 #include "core.h"
28 #include "item.h"
29 #include "cmd.h"
30 #include "port.h"
31 #include "trap.h"
32 #include "emad.h"
33 #include "reg.h"
34 #include "resources.h"
35 
36 static LIST_HEAD(mlxsw_core_driver_list);
37 static DEFINE_SPINLOCK(mlxsw_core_driver_list_lock);
38 
39 static const char mlxsw_core_driver_name[] = "mlxsw_core";
40 
41 static struct workqueue_struct *mlxsw_wq;
42 static struct workqueue_struct *mlxsw_owq;
43 
44 struct mlxsw_core_port {
45 	struct devlink_port devlink_port;
46 	void *port_driver_priv;
47 	u8 local_port;
48 };
49 
50 void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port)
51 {
52 	return mlxsw_core_port->port_driver_priv;
53 }
54 EXPORT_SYMBOL(mlxsw_core_port_driver_priv);
55 
56 static bool mlxsw_core_port_check(struct mlxsw_core_port *mlxsw_core_port)
57 {
58 	return mlxsw_core_port->port_driver_priv != NULL;
59 }
60 
61 struct mlxsw_core {
62 	struct mlxsw_driver *driver;
63 	const struct mlxsw_bus *bus;
64 	void *bus_priv;
65 	const struct mlxsw_bus_info *bus_info;
66 	struct workqueue_struct *emad_wq;
67 	struct list_head rx_listener_list;
68 	struct list_head event_listener_list;
69 	struct {
70 		atomic64_t tid;
71 		struct list_head trans_list;
72 		spinlock_t trans_list_lock; /* protects trans_list writes */
73 		bool use_emad;
74 		bool enable_string_tlv;
75 	} emad;
76 	struct {
77 		u8 *mapping; /* lag_id+port_index to local_port mapping */
78 	} lag;
79 	struct mlxsw_res res;
80 	struct mlxsw_hwmon *hwmon;
81 	struct mlxsw_thermal *thermal;
82 	struct mlxsw_core_port *ports;
83 	unsigned int max_ports;
84 	bool fw_flash_in_progress;
85 	unsigned long driver_priv[];
86 	/* driver_priv has to be always the last item */
87 };
88 
89 #define MLXSW_PORT_MAX_PORTS_DEFAULT	0x40
90 
91 static int mlxsw_ports_init(struct mlxsw_core *mlxsw_core)
92 {
93 	/* Switch ports are numbered from 1 to queried value */
94 	if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_SYSTEM_PORT))
95 		mlxsw_core->max_ports = MLXSW_CORE_RES_GET(mlxsw_core,
96 							   MAX_SYSTEM_PORT) + 1;
97 	else
98 		mlxsw_core->max_ports = MLXSW_PORT_MAX_PORTS_DEFAULT + 1;
99 
100 	mlxsw_core->ports = kcalloc(mlxsw_core->max_ports,
101 				    sizeof(struct mlxsw_core_port), GFP_KERNEL);
102 	if (!mlxsw_core->ports)
103 		return -ENOMEM;
104 
105 	return 0;
106 }
107 
108 static void mlxsw_ports_fini(struct mlxsw_core *mlxsw_core)
109 {
110 	kfree(mlxsw_core->ports);
111 }
112 
113 unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core)
114 {
115 	return mlxsw_core->max_ports;
116 }
117 EXPORT_SYMBOL(mlxsw_core_max_ports);
118 
119 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core)
120 {
121 	return mlxsw_core->driver_priv;
122 }
123 EXPORT_SYMBOL(mlxsw_core_driver_priv);
124 
125 bool mlxsw_core_res_query_enabled(const struct mlxsw_core *mlxsw_core)
126 {
127 	return mlxsw_core->driver->res_query_enabled;
128 }
129 EXPORT_SYMBOL(mlxsw_core_res_query_enabled);
130 
131 bool
132 mlxsw_core_fw_rev_minor_subminor_validate(const struct mlxsw_fw_rev *rev,
133 					  const struct mlxsw_fw_rev *req_rev)
134 {
135 	return rev->minor > req_rev->minor ||
136 	       (rev->minor == req_rev->minor &&
137 		rev->subminor >= req_rev->subminor);
138 }
139 EXPORT_SYMBOL(mlxsw_core_fw_rev_minor_subminor_validate);
140 
141 struct mlxsw_rx_listener_item {
142 	struct list_head list;
143 	struct mlxsw_rx_listener rxl;
144 	void *priv;
145 	bool enabled;
146 };
147 
148 struct mlxsw_event_listener_item {
149 	struct list_head list;
150 	struct mlxsw_event_listener el;
151 	void *priv;
152 };
153 
154 /******************
155  * EMAD processing
156  ******************/
157 
158 /* emad_eth_hdr_dmac
159  * Destination MAC in EMAD's Ethernet header.
160  * Must be set to 01:02:c9:00:00:01
161  */
162 MLXSW_ITEM_BUF(emad, eth_hdr, dmac, 0x00, 6);
163 
164 /* emad_eth_hdr_smac
165  * Source MAC in EMAD's Ethernet header.
166  * Must be set to 00:02:c9:01:02:03
167  */
168 MLXSW_ITEM_BUF(emad, eth_hdr, smac, 0x06, 6);
169 
170 /* emad_eth_hdr_ethertype
171  * Ethertype in EMAD's Ethernet header.
172  * Must be set to 0x8932
173  */
174 MLXSW_ITEM32(emad, eth_hdr, ethertype, 0x0C, 16, 16);
175 
176 /* emad_eth_hdr_mlx_proto
177  * Mellanox protocol.
178  * Must be set to 0x0.
179  */
180 MLXSW_ITEM32(emad, eth_hdr, mlx_proto, 0x0C, 8, 8);
181 
182 /* emad_eth_hdr_ver
183  * Mellanox protocol version.
184  * Must be set to 0x0.
185  */
186 MLXSW_ITEM32(emad, eth_hdr, ver, 0x0C, 4, 4);
187 
188 /* emad_op_tlv_type
189  * Type of the TLV.
190  * Must be set to 0x1 (operation TLV).
191  */
192 MLXSW_ITEM32(emad, op_tlv, type, 0x00, 27, 5);
193 
194 /* emad_op_tlv_len
195  * Length of the operation TLV in u32.
196  * Must be set to 0x4.
197  */
198 MLXSW_ITEM32(emad, op_tlv, len, 0x00, 16, 11);
199 
200 /* emad_op_tlv_dr
201  * Direct route bit. Setting to 1 indicates the EMAD is a direct route
202  * EMAD. DR TLV must follow.
203  *
204  * Note: Currently not supported and must not be set.
205  */
206 MLXSW_ITEM32(emad, op_tlv, dr, 0x00, 15, 1);
207 
208 /* emad_op_tlv_status
209  * Returned status in case of EMAD response. Must be set to 0 in case
210  * of EMAD request.
211  * 0x0 - success
212  * 0x1 - device is busy. Requester should retry
213  * 0x2 - Mellanox protocol version not supported
214  * 0x3 - unknown TLV
215  * 0x4 - register not supported
216  * 0x5 - operation class not supported
217  * 0x6 - EMAD method not supported
218  * 0x7 - bad parameter (e.g. port out of range)
219  * 0x8 - resource not available
220  * 0x9 - message receipt acknowledgment. Requester should retry
221  * 0x70 - internal error
222  */
223 MLXSW_ITEM32(emad, op_tlv, status, 0x00, 8, 7);
224 
225 /* emad_op_tlv_register_id
226  * Register ID of register within register TLV.
227  */
228 MLXSW_ITEM32(emad, op_tlv, register_id, 0x04, 16, 16);
229 
230 /* emad_op_tlv_r
231  * Response bit. Setting to 1 indicates Response, otherwise request.
232  */
233 MLXSW_ITEM32(emad, op_tlv, r, 0x04, 15, 1);
234 
235 /* emad_op_tlv_method
236  * EMAD method type.
237  * 0x1 - query
238  * 0x2 - write
239  * 0x3 - send (currently not supported)
240  * 0x4 - event
241  */
242 MLXSW_ITEM32(emad, op_tlv, method, 0x04, 8, 7);
243 
244 /* emad_op_tlv_class
245  * EMAD operation class. Must be set to 0x1 (REG_ACCESS).
246  */
247 MLXSW_ITEM32(emad, op_tlv, class, 0x04, 0, 8);
248 
249 /* emad_op_tlv_tid
250  * EMAD transaction ID. Used for pairing request and response EMADs.
251  */
252 MLXSW_ITEM64(emad, op_tlv, tid, 0x08, 0, 64);
253 
254 /* emad_string_tlv_type
255  * Type of the TLV.
256  * Must be set to 0x2 (string TLV).
257  */
258 MLXSW_ITEM32(emad, string_tlv, type, 0x00, 27, 5);
259 
260 /* emad_string_tlv_len
261  * Length of the string TLV in u32.
262  */
263 MLXSW_ITEM32(emad, string_tlv, len, 0x00, 16, 11);
264 
265 #define MLXSW_EMAD_STRING_TLV_STRING_LEN 128
266 
267 /* emad_string_tlv_string
268  * String provided by the device's firmware in case of erroneous register access
269  */
270 MLXSW_ITEM_BUF(emad, string_tlv, string, 0x04,
271 	       MLXSW_EMAD_STRING_TLV_STRING_LEN);
272 
273 /* emad_reg_tlv_type
274  * Type of the TLV.
275  * Must be set to 0x3 (register TLV).
276  */
277 MLXSW_ITEM32(emad, reg_tlv, type, 0x00, 27, 5);
278 
279 /* emad_reg_tlv_len
280  * Length of the operation TLV in u32.
281  */
282 MLXSW_ITEM32(emad, reg_tlv, len, 0x00, 16, 11);
283 
284 /* emad_end_tlv_type
285  * Type of the TLV.
286  * Must be set to 0x0 (end TLV).
287  */
288 MLXSW_ITEM32(emad, end_tlv, type, 0x00, 27, 5);
289 
290 /* emad_end_tlv_len
291  * Length of the end TLV in u32.
292  * Must be set to 1.
293  */
294 MLXSW_ITEM32(emad, end_tlv, len, 0x00, 16, 11);
295 
296 enum mlxsw_core_reg_access_type {
297 	MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
298 	MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
299 };
300 
301 static inline const char *
302 mlxsw_core_reg_access_type_str(enum mlxsw_core_reg_access_type type)
303 {
304 	switch (type) {
305 	case MLXSW_CORE_REG_ACCESS_TYPE_QUERY:
306 		return "query";
307 	case MLXSW_CORE_REG_ACCESS_TYPE_WRITE:
308 		return "write";
309 	}
310 	BUG();
311 }
312 
313 static void mlxsw_emad_pack_end_tlv(char *end_tlv)
314 {
315 	mlxsw_emad_end_tlv_type_set(end_tlv, MLXSW_EMAD_TLV_TYPE_END);
316 	mlxsw_emad_end_tlv_len_set(end_tlv, MLXSW_EMAD_END_TLV_LEN);
317 }
318 
319 static void mlxsw_emad_pack_reg_tlv(char *reg_tlv,
320 				    const struct mlxsw_reg_info *reg,
321 				    char *payload)
322 {
323 	mlxsw_emad_reg_tlv_type_set(reg_tlv, MLXSW_EMAD_TLV_TYPE_REG);
324 	mlxsw_emad_reg_tlv_len_set(reg_tlv, reg->len / sizeof(u32) + 1);
325 	memcpy(reg_tlv + sizeof(u32), payload, reg->len);
326 }
327 
328 static void mlxsw_emad_pack_string_tlv(char *string_tlv)
329 {
330 	mlxsw_emad_string_tlv_type_set(string_tlv, MLXSW_EMAD_TLV_TYPE_STRING);
331 	mlxsw_emad_string_tlv_len_set(string_tlv, MLXSW_EMAD_STRING_TLV_LEN);
332 }
333 
334 static void mlxsw_emad_pack_op_tlv(char *op_tlv,
335 				   const struct mlxsw_reg_info *reg,
336 				   enum mlxsw_core_reg_access_type type,
337 				   u64 tid)
338 {
339 	mlxsw_emad_op_tlv_type_set(op_tlv, MLXSW_EMAD_TLV_TYPE_OP);
340 	mlxsw_emad_op_tlv_len_set(op_tlv, MLXSW_EMAD_OP_TLV_LEN);
341 	mlxsw_emad_op_tlv_dr_set(op_tlv, 0);
342 	mlxsw_emad_op_tlv_status_set(op_tlv, 0);
343 	mlxsw_emad_op_tlv_register_id_set(op_tlv, reg->id);
344 	mlxsw_emad_op_tlv_r_set(op_tlv, MLXSW_EMAD_OP_TLV_REQUEST);
345 	if (type == MLXSW_CORE_REG_ACCESS_TYPE_QUERY)
346 		mlxsw_emad_op_tlv_method_set(op_tlv,
347 					     MLXSW_EMAD_OP_TLV_METHOD_QUERY);
348 	else
349 		mlxsw_emad_op_tlv_method_set(op_tlv,
350 					     MLXSW_EMAD_OP_TLV_METHOD_WRITE);
351 	mlxsw_emad_op_tlv_class_set(op_tlv,
352 				    MLXSW_EMAD_OP_TLV_CLASS_REG_ACCESS);
353 	mlxsw_emad_op_tlv_tid_set(op_tlv, tid);
354 }
355 
356 static int mlxsw_emad_construct_eth_hdr(struct sk_buff *skb)
357 {
358 	char *eth_hdr = skb_push(skb, MLXSW_EMAD_ETH_HDR_LEN);
359 
360 	mlxsw_emad_eth_hdr_dmac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_DMAC);
361 	mlxsw_emad_eth_hdr_smac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_SMAC);
362 	mlxsw_emad_eth_hdr_ethertype_set(eth_hdr, MLXSW_EMAD_EH_ETHERTYPE);
363 	mlxsw_emad_eth_hdr_mlx_proto_set(eth_hdr, MLXSW_EMAD_EH_MLX_PROTO);
364 	mlxsw_emad_eth_hdr_ver_set(eth_hdr, MLXSW_EMAD_EH_PROTO_VERSION);
365 
366 	skb_reset_mac_header(skb);
367 
368 	return 0;
369 }
370 
371 static void mlxsw_emad_construct(struct sk_buff *skb,
372 				 const struct mlxsw_reg_info *reg,
373 				 char *payload,
374 				 enum mlxsw_core_reg_access_type type,
375 				 u64 tid, bool enable_string_tlv)
376 {
377 	char *buf;
378 
379 	buf = skb_push(skb, MLXSW_EMAD_END_TLV_LEN * sizeof(u32));
380 	mlxsw_emad_pack_end_tlv(buf);
381 
382 	buf = skb_push(skb, reg->len + sizeof(u32));
383 	mlxsw_emad_pack_reg_tlv(buf, reg, payload);
384 
385 	if (enable_string_tlv) {
386 		buf = skb_push(skb, MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32));
387 		mlxsw_emad_pack_string_tlv(buf);
388 	}
389 
390 	buf = skb_push(skb, MLXSW_EMAD_OP_TLV_LEN * sizeof(u32));
391 	mlxsw_emad_pack_op_tlv(buf, reg, type, tid);
392 
393 	mlxsw_emad_construct_eth_hdr(skb);
394 }
395 
396 struct mlxsw_emad_tlv_offsets {
397 	u16 op_tlv;
398 	u16 string_tlv;
399 	u16 reg_tlv;
400 };
401 
402 static bool mlxsw_emad_tlv_is_string_tlv(const char *tlv)
403 {
404 	u8 tlv_type = mlxsw_emad_string_tlv_type_get(tlv);
405 
406 	return tlv_type == MLXSW_EMAD_TLV_TYPE_STRING;
407 }
408 
409 static void mlxsw_emad_tlv_parse(struct sk_buff *skb)
410 {
411 	struct mlxsw_emad_tlv_offsets *offsets =
412 		(struct mlxsw_emad_tlv_offsets *) skb->cb;
413 
414 	offsets->op_tlv = MLXSW_EMAD_ETH_HDR_LEN;
415 	offsets->string_tlv = 0;
416 	offsets->reg_tlv = MLXSW_EMAD_ETH_HDR_LEN +
417 			   MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
418 
419 	/* If string TLV is present, it must come after the operation TLV. */
420 	if (mlxsw_emad_tlv_is_string_tlv(skb->data + offsets->reg_tlv)) {
421 		offsets->string_tlv = offsets->reg_tlv;
422 		offsets->reg_tlv += MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32);
423 	}
424 }
425 
426 static char *mlxsw_emad_op_tlv(const struct sk_buff *skb)
427 {
428 	struct mlxsw_emad_tlv_offsets *offsets =
429 		(struct mlxsw_emad_tlv_offsets *) skb->cb;
430 
431 	return ((char *) (skb->data + offsets->op_tlv));
432 }
433 
434 static char *mlxsw_emad_string_tlv(const struct sk_buff *skb)
435 {
436 	struct mlxsw_emad_tlv_offsets *offsets =
437 		(struct mlxsw_emad_tlv_offsets *) skb->cb;
438 
439 	if (!offsets->string_tlv)
440 		return NULL;
441 
442 	return ((char *) (skb->data + offsets->string_tlv));
443 }
444 
445 static char *mlxsw_emad_reg_tlv(const struct sk_buff *skb)
446 {
447 	struct mlxsw_emad_tlv_offsets *offsets =
448 		(struct mlxsw_emad_tlv_offsets *) skb->cb;
449 
450 	return ((char *) (skb->data + offsets->reg_tlv));
451 }
452 
453 static char *mlxsw_emad_reg_payload(const char *reg_tlv)
454 {
455 	return ((char *) (reg_tlv + sizeof(u32)));
456 }
457 
458 static char *mlxsw_emad_reg_payload_cmd(const char *mbox)
459 {
460 	return ((char *) (mbox + (MLXSW_EMAD_OP_TLV_LEN + 1) * sizeof(u32)));
461 }
462 
463 static u64 mlxsw_emad_get_tid(const struct sk_buff *skb)
464 {
465 	char *op_tlv;
466 
467 	op_tlv = mlxsw_emad_op_tlv(skb);
468 	return mlxsw_emad_op_tlv_tid_get(op_tlv);
469 }
470 
471 static bool mlxsw_emad_is_resp(const struct sk_buff *skb)
472 {
473 	char *op_tlv;
474 
475 	op_tlv = mlxsw_emad_op_tlv(skb);
476 	return (mlxsw_emad_op_tlv_r_get(op_tlv) == MLXSW_EMAD_OP_TLV_RESPONSE);
477 }
478 
479 static int mlxsw_emad_process_status(char *op_tlv,
480 				     enum mlxsw_emad_op_tlv_status *p_status)
481 {
482 	*p_status = mlxsw_emad_op_tlv_status_get(op_tlv);
483 
484 	switch (*p_status) {
485 	case MLXSW_EMAD_OP_TLV_STATUS_SUCCESS:
486 		return 0;
487 	case MLXSW_EMAD_OP_TLV_STATUS_BUSY:
488 	case MLXSW_EMAD_OP_TLV_STATUS_MESSAGE_RECEIPT_ACK:
489 		return -EAGAIN;
490 	case MLXSW_EMAD_OP_TLV_STATUS_VERSION_NOT_SUPPORTED:
491 	case MLXSW_EMAD_OP_TLV_STATUS_UNKNOWN_TLV:
492 	case MLXSW_EMAD_OP_TLV_STATUS_REGISTER_NOT_SUPPORTED:
493 	case MLXSW_EMAD_OP_TLV_STATUS_CLASS_NOT_SUPPORTED:
494 	case MLXSW_EMAD_OP_TLV_STATUS_METHOD_NOT_SUPPORTED:
495 	case MLXSW_EMAD_OP_TLV_STATUS_BAD_PARAMETER:
496 	case MLXSW_EMAD_OP_TLV_STATUS_RESOURCE_NOT_AVAILABLE:
497 	case MLXSW_EMAD_OP_TLV_STATUS_INTERNAL_ERROR:
498 	default:
499 		return -EIO;
500 	}
501 }
502 
503 static int
504 mlxsw_emad_process_status_skb(struct sk_buff *skb,
505 			      enum mlxsw_emad_op_tlv_status *p_status)
506 {
507 	return mlxsw_emad_process_status(mlxsw_emad_op_tlv(skb), p_status);
508 }
509 
510 struct mlxsw_reg_trans {
511 	struct list_head list;
512 	struct list_head bulk_list;
513 	struct mlxsw_core *core;
514 	struct sk_buff *tx_skb;
515 	struct mlxsw_tx_info tx_info;
516 	struct delayed_work timeout_dw;
517 	unsigned int retries;
518 	u64 tid;
519 	struct completion completion;
520 	atomic_t active;
521 	mlxsw_reg_trans_cb_t *cb;
522 	unsigned long cb_priv;
523 	const struct mlxsw_reg_info *reg;
524 	enum mlxsw_core_reg_access_type type;
525 	int err;
526 	char *emad_err_string;
527 	enum mlxsw_emad_op_tlv_status emad_status;
528 	struct rcu_head rcu;
529 };
530 
531 static void mlxsw_emad_process_string_tlv(const struct sk_buff *skb,
532 					  struct mlxsw_reg_trans *trans)
533 {
534 	char *string_tlv;
535 	char *string;
536 
537 	string_tlv = mlxsw_emad_string_tlv(skb);
538 	if (!string_tlv)
539 		return;
540 
541 	trans->emad_err_string = kzalloc(MLXSW_EMAD_STRING_TLV_STRING_LEN,
542 					 GFP_ATOMIC);
543 	if (!trans->emad_err_string)
544 		return;
545 
546 	string = mlxsw_emad_string_tlv_string_data(string_tlv);
547 	strlcpy(trans->emad_err_string, string,
548 		MLXSW_EMAD_STRING_TLV_STRING_LEN);
549 }
550 
551 #define MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS	3000
552 #define MLXSW_EMAD_TIMEOUT_MS			200
553 
554 static void mlxsw_emad_trans_timeout_schedule(struct mlxsw_reg_trans *trans)
555 {
556 	unsigned long timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_MS);
557 
558 	if (trans->core->fw_flash_in_progress)
559 		timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS);
560 
561 	queue_delayed_work(trans->core->emad_wq, &trans->timeout_dw, timeout);
562 }
563 
564 static int mlxsw_emad_transmit(struct mlxsw_core *mlxsw_core,
565 			       struct mlxsw_reg_trans *trans)
566 {
567 	struct sk_buff *skb;
568 	int err;
569 
570 	skb = skb_copy(trans->tx_skb, GFP_KERNEL);
571 	if (!skb)
572 		return -ENOMEM;
573 
574 	trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), false, 0,
575 			    skb->data + mlxsw_core->driver->txhdr_len,
576 			    skb->len - mlxsw_core->driver->txhdr_len);
577 
578 	atomic_set(&trans->active, 1);
579 	err = mlxsw_core_skb_transmit(mlxsw_core, skb, &trans->tx_info);
580 	if (err) {
581 		dev_kfree_skb(skb);
582 		return err;
583 	}
584 	mlxsw_emad_trans_timeout_schedule(trans);
585 	return 0;
586 }
587 
588 static void mlxsw_emad_trans_finish(struct mlxsw_reg_trans *trans, int err)
589 {
590 	struct mlxsw_core *mlxsw_core = trans->core;
591 
592 	dev_kfree_skb(trans->tx_skb);
593 	spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
594 	list_del_rcu(&trans->list);
595 	spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
596 	trans->err = err;
597 	complete(&trans->completion);
598 }
599 
600 static void mlxsw_emad_transmit_retry(struct mlxsw_core *mlxsw_core,
601 				      struct mlxsw_reg_trans *trans)
602 {
603 	int err;
604 
605 	if (trans->retries < MLXSW_EMAD_MAX_RETRY) {
606 		trans->retries++;
607 		err = mlxsw_emad_transmit(trans->core, trans);
608 		if (err == 0)
609 			return;
610 	} else {
611 		err = -EIO;
612 	}
613 	mlxsw_emad_trans_finish(trans, err);
614 }
615 
616 static void mlxsw_emad_trans_timeout_work(struct work_struct *work)
617 {
618 	struct mlxsw_reg_trans *trans = container_of(work,
619 						     struct mlxsw_reg_trans,
620 						     timeout_dw.work);
621 
622 	if (!atomic_dec_and_test(&trans->active))
623 		return;
624 
625 	mlxsw_emad_transmit_retry(trans->core, trans);
626 }
627 
628 static void mlxsw_emad_process_response(struct mlxsw_core *mlxsw_core,
629 					struct mlxsw_reg_trans *trans,
630 					struct sk_buff *skb)
631 {
632 	int err;
633 
634 	if (!atomic_dec_and_test(&trans->active))
635 		return;
636 
637 	err = mlxsw_emad_process_status_skb(skb, &trans->emad_status);
638 	if (err == -EAGAIN) {
639 		mlxsw_emad_transmit_retry(mlxsw_core, trans);
640 	} else {
641 		if (err == 0) {
642 			char *reg_tlv = mlxsw_emad_reg_tlv(skb);
643 
644 			if (trans->cb)
645 				trans->cb(mlxsw_core,
646 					  mlxsw_emad_reg_payload(reg_tlv),
647 					  trans->reg->len, trans->cb_priv);
648 		} else {
649 			mlxsw_emad_process_string_tlv(skb, trans);
650 		}
651 		mlxsw_emad_trans_finish(trans, err);
652 	}
653 }
654 
655 /* called with rcu read lock held */
656 static void mlxsw_emad_rx_listener_func(struct sk_buff *skb, u8 local_port,
657 					void *priv)
658 {
659 	struct mlxsw_core *mlxsw_core = priv;
660 	struct mlxsw_reg_trans *trans;
661 
662 	trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), true, 0,
663 			    skb->data, skb->len);
664 
665 	mlxsw_emad_tlv_parse(skb);
666 
667 	if (!mlxsw_emad_is_resp(skb))
668 		goto free_skb;
669 
670 	list_for_each_entry_rcu(trans, &mlxsw_core->emad.trans_list, list) {
671 		if (mlxsw_emad_get_tid(skb) == trans->tid) {
672 			mlxsw_emad_process_response(mlxsw_core, trans, skb);
673 			break;
674 		}
675 	}
676 
677 free_skb:
678 	dev_kfree_skb(skb);
679 }
680 
681 static const struct mlxsw_listener mlxsw_emad_rx_listener =
682 	MLXSW_RXL(mlxsw_emad_rx_listener_func, ETHEMAD, TRAP_TO_CPU, false,
683 		  EMAD, DISCARD);
684 
685 static int mlxsw_emad_init(struct mlxsw_core *mlxsw_core)
686 {
687 	struct workqueue_struct *emad_wq;
688 	u64 tid;
689 	int err;
690 
691 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
692 		return 0;
693 
694 	emad_wq = alloc_workqueue("mlxsw_core_emad", 0, 0);
695 	if (!emad_wq)
696 		return -ENOMEM;
697 	mlxsw_core->emad_wq = emad_wq;
698 
699 	/* Set the upper 32 bits of the transaction ID field to a random
700 	 * number. This allows us to discard EMADs addressed to other
701 	 * devices.
702 	 */
703 	get_random_bytes(&tid, 4);
704 	tid <<= 32;
705 	atomic64_set(&mlxsw_core->emad.tid, tid);
706 
707 	INIT_LIST_HEAD(&mlxsw_core->emad.trans_list);
708 	spin_lock_init(&mlxsw_core->emad.trans_list_lock);
709 
710 	err = mlxsw_core_trap_register(mlxsw_core, &mlxsw_emad_rx_listener,
711 				       mlxsw_core);
712 	if (err)
713 		goto err_trap_register;
714 
715 	err = mlxsw_core->driver->basic_trap_groups_set(mlxsw_core);
716 	if (err)
717 		goto err_emad_trap_set;
718 	mlxsw_core->emad.use_emad = true;
719 
720 	return 0;
721 
722 err_emad_trap_set:
723 	mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
724 				   mlxsw_core);
725 err_trap_register:
726 	destroy_workqueue(mlxsw_core->emad_wq);
727 	return err;
728 }
729 
730 static void mlxsw_emad_fini(struct mlxsw_core *mlxsw_core)
731 {
732 
733 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
734 		return;
735 
736 	mlxsw_core->emad.use_emad = false;
737 	mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
738 				   mlxsw_core);
739 	destroy_workqueue(mlxsw_core->emad_wq);
740 }
741 
742 static struct sk_buff *mlxsw_emad_alloc(const struct mlxsw_core *mlxsw_core,
743 					u16 reg_len, bool enable_string_tlv)
744 {
745 	struct sk_buff *skb;
746 	u16 emad_len;
747 
748 	emad_len = (reg_len + sizeof(u32) + MLXSW_EMAD_ETH_HDR_LEN +
749 		    (MLXSW_EMAD_OP_TLV_LEN + MLXSW_EMAD_END_TLV_LEN) *
750 		    sizeof(u32) + mlxsw_core->driver->txhdr_len);
751 	if (enable_string_tlv)
752 		emad_len += MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32);
753 	if (emad_len > MLXSW_EMAD_MAX_FRAME_LEN)
754 		return NULL;
755 
756 	skb = netdev_alloc_skb(NULL, emad_len);
757 	if (!skb)
758 		return NULL;
759 	memset(skb->data, 0, emad_len);
760 	skb_reserve(skb, emad_len);
761 
762 	return skb;
763 }
764 
765 static int mlxsw_emad_reg_access(struct mlxsw_core *mlxsw_core,
766 				 const struct mlxsw_reg_info *reg,
767 				 char *payload,
768 				 enum mlxsw_core_reg_access_type type,
769 				 struct mlxsw_reg_trans *trans,
770 				 struct list_head *bulk_list,
771 				 mlxsw_reg_trans_cb_t *cb,
772 				 unsigned long cb_priv, u64 tid)
773 {
774 	bool enable_string_tlv;
775 	struct sk_buff *skb;
776 	int err;
777 
778 	dev_dbg(mlxsw_core->bus_info->dev, "EMAD reg access (tid=%llx,reg_id=%x(%s),type=%s)\n",
779 		tid, reg->id, mlxsw_reg_id_str(reg->id),
780 		mlxsw_core_reg_access_type_str(type));
781 
782 	/* Since this can be changed during emad_reg_access, read it once and
783 	 * use the value all the way.
784 	 */
785 	enable_string_tlv = mlxsw_core->emad.enable_string_tlv;
786 
787 	skb = mlxsw_emad_alloc(mlxsw_core, reg->len, enable_string_tlv);
788 	if (!skb)
789 		return -ENOMEM;
790 
791 	list_add_tail(&trans->bulk_list, bulk_list);
792 	trans->core = mlxsw_core;
793 	trans->tx_skb = skb;
794 	trans->tx_info.local_port = MLXSW_PORT_CPU_PORT;
795 	trans->tx_info.is_emad = true;
796 	INIT_DELAYED_WORK(&trans->timeout_dw, mlxsw_emad_trans_timeout_work);
797 	trans->tid = tid;
798 	init_completion(&trans->completion);
799 	trans->cb = cb;
800 	trans->cb_priv = cb_priv;
801 	trans->reg = reg;
802 	trans->type = type;
803 
804 	mlxsw_emad_construct(skb, reg, payload, type, trans->tid,
805 			     enable_string_tlv);
806 	mlxsw_core->driver->txhdr_construct(skb, &trans->tx_info);
807 
808 	spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
809 	list_add_tail_rcu(&trans->list, &mlxsw_core->emad.trans_list);
810 	spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
811 	err = mlxsw_emad_transmit(mlxsw_core, trans);
812 	if (err)
813 		goto err_out;
814 	return 0;
815 
816 err_out:
817 	spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
818 	list_del_rcu(&trans->list);
819 	spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
820 	list_del(&trans->bulk_list);
821 	dev_kfree_skb(trans->tx_skb);
822 	return err;
823 }
824 
825 /*****************
826  * Core functions
827  *****************/
828 
829 int mlxsw_core_driver_register(struct mlxsw_driver *mlxsw_driver)
830 {
831 	spin_lock(&mlxsw_core_driver_list_lock);
832 	list_add_tail(&mlxsw_driver->list, &mlxsw_core_driver_list);
833 	spin_unlock(&mlxsw_core_driver_list_lock);
834 	return 0;
835 }
836 EXPORT_SYMBOL(mlxsw_core_driver_register);
837 
838 void mlxsw_core_driver_unregister(struct mlxsw_driver *mlxsw_driver)
839 {
840 	spin_lock(&mlxsw_core_driver_list_lock);
841 	list_del(&mlxsw_driver->list);
842 	spin_unlock(&mlxsw_core_driver_list_lock);
843 }
844 EXPORT_SYMBOL(mlxsw_core_driver_unregister);
845 
846 static struct mlxsw_driver *__driver_find(const char *kind)
847 {
848 	struct mlxsw_driver *mlxsw_driver;
849 
850 	list_for_each_entry(mlxsw_driver, &mlxsw_core_driver_list, list) {
851 		if (strcmp(mlxsw_driver->kind, kind) == 0)
852 			return mlxsw_driver;
853 	}
854 	return NULL;
855 }
856 
857 static struct mlxsw_driver *mlxsw_core_driver_get(const char *kind)
858 {
859 	struct mlxsw_driver *mlxsw_driver;
860 
861 	spin_lock(&mlxsw_core_driver_list_lock);
862 	mlxsw_driver = __driver_find(kind);
863 	spin_unlock(&mlxsw_core_driver_list_lock);
864 	return mlxsw_driver;
865 }
866 
867 static int mlxsw_devlink_port_split(struct devlink *devlink,
868 				    unsigned int port_index,
869 				    unsigned int count,
870 				    struct netlink_ext_ack *extack)
871 {
872 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
873 
874 	if (port_index >= mlxsw_core->max_ports) {
875 		NL_SET_ERR_MSG_MOD(extack, "Port index exceeds maximum number of ports");
876 		return -EINVAL;
877 	}
878 	if (!mlxsw_core->driver->port_split)
879 		return -EOPNOTSUPP;
880 	return mlxsw_core->driver->port_split(mlxsw_core, port_index, count,
881 					      extack);
882 }
883 
884 static int mlxsw_devlink_port_unsplit(struct devlink *devlink,
885 				      unsigned int port_index,
886 				      struct netlink_ext_ack *extack)
887 {
888 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
889 
890 	if (port_index >= mlxsw_core->max_ports) {
891 		NL_SET_ERR_MSG_MOD(extack, "Port index exceeds maximum number of ports");
892 		return -EINVAL;
893 	}
894 	if (!mlxsw_core->driver->port_unsplit)
895 		return -EOPNOTSUPP;
896 	return mlxsw_core->driver->port_unsplit(mlxsw_core, port_index,
897 						extack);
898 }
899 
900 static int
901 mlxsw_devlink_sb_pool_get(struct devlink *devlink,
902 			  unsigned int sb_index, u16 pool_index,
903 			  struct devlink_sb_pool_info *pool_info)
904 {
905 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
906 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
907 
908 	if (!mlxsw_driver->sb_pool_get)
909 		return -EOPNOTSUPP;
910 	return mlxsw_driver->sb_pool_get(mlxsw_core, sb_index,
911 					 pool_index, pool_info);
912 }
913 
914 static int
915 mlxsw_devlink_sb_pool_set(struct devlink *devlink,
916 			  unsigned int sb_index, u16 pool_index, u32 size,
917 			  enum devlink_sb_threshold_type threshold_type,
918 			  struct netlink_ext_ack *extack)
919 {
920 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
921 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
922 
923 	if (!mlxsw_driver->sb_pool_set)
924 		return -EOPNOTSUPP;
925 	return mlxsw_driver->sb_pool_set(mlxsw_core, sb_index,
926 					 pool_index, size, threshold_type,
927 					 extack);
928 }
929 
930 static void *__dl_port(struct devlink_port *devlink_port)
931 {
932 	return container_of(devlink_port, struct mlxsw_core_port, devlink_port);
933 }
934 
935 static int mlxsw_devlink_port_type_set(struct devlink_port *devlink_port,
936 				       enum devlink_port_type port_type)
937 {
938 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
939 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
940 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
941 
942 	if (!mlxsw_driver->port_type_set)
943 		return -EOPNOTSUPP;
944 
945 	return mlxsw_driver->port_type_set(mlxsw_core,
946 					   mlxsw_core_port->local_port,
947 					   port_type);
948 }
949 
950 static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port,
951 					  unsigned int sb_index, u16 pool_index,
952 					  u32 *p_threshold)
953 {
954 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
955 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
956 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
957 
958 	if (!mlxsw_driver->sb_port_pool_get ||
959 	    !mlxsw_core_port_check(mlxsw_core_port))
960 		return -EOPNOTSUPP;
961 	return mlxsw_driver->sb_port_pool_get(mlxsw_core_port, sb_index,
962 					      pool_index, p_threshold);
963 }
964 
965 static int mlxsw_devlink_sb_port_pool_set(struct devlink_port *devlink_port,
966 					  unsigned int sb_index, u16 pool_index,
967 					  u32 threshold,
968 					  struct netlink_ext_ack *extack)
969 {
970 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
971 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
972 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
973 
974 	if (!mlxsw_driver->sb_port_pool_set ||
975 	    !mlxsw_core_port_check(mlxsw_core_port))
976 		return -EOPNOTSUPP;
977 	return mlxsw_driver->sb_port_pool_set(mlxsw_core_port, sb_index,
978 					      pool_index, threshold, extack);
979 }
980 
981 static int
982 mlxsw_devlink_sb_tc_pool_bind_get(struct devlink_port *devlink_port,
983 				  unsigned int sb_index, u16 tc_index,
984 				  enum devlink_sb_pool_type pool_type,
985 				  u16 *p_pool_index, u32 *p_threshold)
986 {
987 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
988 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
989 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
990 
991 	if (!mlxsw_driver->sb_tc_pool_bind_get ||
992 	    !mlxsw_core_port_check(mlxsw_core_port))
993 		return -EOPNOTSUPP;
994 	return mlxsw_driver->sb_tc_pool_bind_get(mlxsw_core_port, sb_index,
995 						 tc_index, pool_type,
996 						 p_pool_index, p_threshold);
997 }
998 
999 static int
1000 mlxsw_devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
1001 				  unsigned int sb_index, u16 tc_index,
1002 				  enum devlink_sb_pool_type pool_type,
1003 				  u16 pool_index, u32 threshold,
1004 				  struct netlink_ext_ack *extack)
1005 {
1006 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1007 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1008 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1009 
1010 	if (!mlxsw_driver->sb_tc_pool_bind_set ||
1011 	    !mlxsw_core_port_check(mlxsw_core_port))
1012 		return -EOPNOTSUPP;
1013 	return mlxsw_driver->sb_tc_pool_bind_set(mlxsw_core_port, sb_index,
1014 						 tc_index, pool_type,
1015 						 pool_index, threshold, extack);
1016 }
1017 
1018 static int mlxsw_devlink_sb_occ_snapshot(struct devlink *devlink,
1019 					 unsigned int sb_index)
1020 {
1021 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1022 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1023 
1024 	if (!mlxsw_driver->sb_occ_snapshot)
1025 		return -EOPNOTSUPP;
1026 	return mlxsw_driver->sb_occ_snapshot(mlxsw_core, sb_index);
1027 }
1028 
1029 static int mlxsw_devlink_sb_occ_max_clear(struct devlink *devlink,
1030 					  unsigned int sb_index)
1031 {
1032 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1033 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1034 
1035 	if (!mlxsw_driver->sb_occ_max_clear)
1036 		return -EOPNOTSUPP;
1037 	return mlxsw_driver->sb_occ_max_clear(mlxsw_core, sb_index);
1038 }
1039 
1040 static int
1041 mlxsw_devlink_sb_occ_port_pool_get(struct devlink_port *devlink_port,
1042 				   unsigned int sb_index, u16 pool_index,
1043 				   u32 *p_cur, u32 *p_max)
1044 {
1045 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1046 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1047 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1048 
1049 	if (!mlxsw_driver->sb_occ_port_pool_get ||
1050 	    !mlxsw_core_port_check(mlxsw_core_port))
1051 		return -EOPNOTSUPP;
1052 	return mlxsw_driver->sb_occ_port_pool_get(mlxsw_core_port, sb_index,
1053 						  pool_index, p_cur, p_max);
1054 }
1055 
1056 static int
1057 mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port,
1058 				      unsigned int sb_index, u16 tc_index,
1059 				      enum devlink_sb_pool_type pool_type,
1060 				      u32 *p_cur, u32 *p_max)
1061 {
1062 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1063 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1064 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1065 
1066 	if (!mlxsw_driver->sb_occ_tc_port_bind_get ||
1067 	    !mlxsw_core_port_check(mlxsw_core_port))
1068 		return -EOPNOTSUPP;
1069 	return mlxsw_driver->sb_occ_tc_port_bind_get(mlxsw_core_port,
1070 						     sb_index, tc_index,
1071 						     pool_type, p_cur, p_max);
1072 }
1073 
1074 static int
1075 mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req,
1076 		       struct netlink_ext_ack *extack)
1077 {
1078 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1079 	char fw_info_psid[MLXSW_REG_MGIR_FW_INFO_PSID_SIZE];
1080 	u32 hw_rev, fw_major, fw_minor, fw_sub_minor;
1081 	char mgir_pl[MLXSW_REG_MGIR_LEN];
1082 	char buf[32];
1083 	int err;
1084 
1085 	err = devlink_info_driver_name_put(req,
1086 					   mlxsw_core->bus_info->device_kind);
1087 	if (err)
1088 		return err;
1089 
1090 	mlxsw_reg_mgir_pack(mgir_pl);
1091 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mgir), mgir_pl);
1092 	if (err)
1093 		return err;
1094 	mlxsw_reg_mgir_unpack(mgir_pl, &hw_rev, fw_info_psid, &fw_major,
1095 			      &fw_minor, &fw_sub_minor);
1096 
1097 	sprintf(buf, "%X", hw_rev);
1098 	err = devlink_info_version_fixed_put(req, "hw.revision", buf);
1099 	if (err)
1100 		return err;
1101 
1102 	err = devlink_info_version_fixed_put(req, "fw.psid", fw_info_psid);
1103 	if (err)
1104 		return err;
1105 
1106 	sprintf(buf, "%d.%d.%d", fw_major, fw_minor, fw_sub_minor);
1107 	err = devlink_info_version_running_put(req, "fw.version", buf);
1108 	if (err)
1109 		return err;
1110 
1111 	return 0;
1112 }
1113 
1114 static int
1115 mlxsw_devlink_core_bus_device_reload_down(struct devlink *devlink,
1116 					  bool netns_change,
1117 					  struct netlink_ext_ack *extack)
1118 {
1119 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1120 
1121 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_RESET))
1122 		return -EOPNOTSUPP;
1123 
1124 	mlxsw_core_bus_device_unregister(mlxsw_core, true);
1125 	return 0;
1126 }
1127 
1128 static int
1129 mlxsw_devlink_core_bus_device_reload_up(struct devlink *devlink,
1130 					struct netlink_ext_ack *extack)
1131 {
1132 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1133 
1134 	return mlxsw_core_bus_device_register(mlxsw_core->bus_info,
1135 					      mlxsw_core->bus,
1136 					      mlxsw_core->bus_priv, true,
1137 					      devlink, extack);
1138 }
1139 
1140 static int mlxsw_devlink_flash_update(struct devlink *devlink,
1141 				      const char *file_name,
1142 				      const char *component,
1143 				      struct netlink_ext_ack *extack)
1144 {
1145 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1146 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1147 
1148 	if (!mlxsw_driver->flash_update)
1149 		return -EOPNOTSUPP;
1150 	return mlxsw_driver->flash_update(mlxsw_core, file_name,
1151 					  component, extack);
1152 }
1153 
1154 static int mlxsw_devlink_trap_init(struct devlink *devlink,
1155 				   const struct devlink_trap *trap,
1156 				   void *trap_ctx)
1157 {
1158 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1159 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1160 
1161 	if (!mlxsw_driver->trap_init)
1162 		return -EOPNOTSUPP;
1163 	return mlxsw_driver->trap_init(mlxsw_core, trap, trap_ctx);
1164 }
1165 
1166 static void mlxsw_devlink_trap_fini(struct devlink *devlink,
1167 				    const struct devlink_trap *trap,
1168 				    void *trap_ctx)
1169 {
1170 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1171 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1172 
1173 	if (!mlxsw_driver->trap_fini)
1174 		return;
1175 	mlxsw_driver->trap_fini(mlxsw_core, trap, trap_ctx);
1176 }
1177 
1178 static int mlxsw_devlink_trap_action_set(struct devlink *devlink,
1179 					 const struct devlink_trap *trap,
1180 					 enum devlink_trap_action action,
1181 					 struct netlink_ext_ack *extack)
1182 {
1183 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1184 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1185 
1186 	if (!mlxsw_driver->trap_action_set)
1187 		return -EOPNOTSUPP;
1188 	return mlxsw_driver->trap_action_set(mlxsw_core, trap, action, extack);
1189 }
1190 
1191 static int
1192 mlxsw_devlink_trap_group_init(struct devlink *devlink,
1193 			      const struct devlink_trap_group *group)
1194 {
1195 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1196 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1197 
1198 	if (!mlxsw_driver->trap_group_init)
1199 		return -EOPNOTSUPP;
1200 	return mlxsw_driver->trap_group_init(mlxsw_core, group);
1201 }
1202 
1203 static int
1204 mlxsw_devlink_trap_group_set(struct devlink *devlink,
1205 			     const struct devlink_trap_group *group,
1206 			     const struct devlink_trap_policer *policer,
1207 			     struct netlink_ext_ack *extack)
1208 {
1209 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1210 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1211 
1212 	if (!mlxsw_driver->trap_group_set)
1213 		return -EOPNOTSUPP;
1214 	return mlxsw_driver->trap_group_set(mlxsw_core, group, policer, extack);
1215 }
1216 
1217 static int
1218 mlxsw_devlink_trap_policer_init(struct devlink *devlink,
1219 				const struct devlink_trap_policer *policer)
1220 {
1221 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1222 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1223 
1224 	if (!mlxsw_driver->trap_policer_init)
1225 		return -EOPNOTSUPP;
1226 	return mlxsw_driver->trap_policer_init(mlxsw_core, policer);
1227 }
1228 
1229 static void
1230 mlxsw_devlink_trap_policer_fini(struct devlink *devlink,
1231 				const struct devlink_trap_policer *policer)
1232 {
1233 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1234 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1235 
1236 	if (!mlxsw_driver->trap_policer_fini)
1237 		return;
1238 	mlxsw_driver->trap_policer_fini(mlxsw_core, policer);
1239 }
1240 
1241 static int
1242 mlxsw_devlink_trap_policer_set(struct devlink *devlink,
1243 			       const struct devlink_trap_policer *policer,
1244 			       u64 rate, u64 burst,
1245 			       struct netlink_ext_ack *extack)
1246 {
1247 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1248 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1249 
1250 	if (!mlxsw_driver->trap_policer_set)
1251 		return -EOPNOTSUPP;
1252 	return mlxsw_driver->trap_policer_set(mlxsw_core, policer, rate, burst,
1253 					      extack);
1254 }
1255 
1256 static int
1257 mlxsw_devlink_trap_policer_counter_get(struct devlink *devlink,
1258 				       const struct devlink_trap_policer *policer,
1259 				       u64 *p_drops)
1260 {
1261 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1262 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1263 
1264 	if (!mlxsw_driver->trap_policer_counter_get)
1265 		return -EOPNOTSUPP;
1266 	return mlxsw_driver->trap_policer_counter_get(mlxsw_core, policer,
1267 						      p_drops);
1268 }
1269 
1270 static const struct devlink_ops mlxsw_devlink_ops = {
1271 	.reload_down		= mlxsw_devlink_core_bus_device_reload_down,
1272 	.reload_up		= mlxsw_devlink_core_bus_device_reload_up,
1273 	.port_type_set			= mlxsw_devlink_port_type_set,
1274 	.port_split			= mlxsw_devlink_port_split,
1275 	.port_unsplit			= mlxsw_devlink_port_unsplit,
1276 	.sb_pool_get			= mlxsw_devlink_sb_pool_get,
1277 	.sb_pool_set			= mlxsw_devlink_sb_pool_set,
1278 	.sb_port_pool_get		= mlxsw_devlink_sb_port_pool_get,
1279 	.sb_port_pool_set		= mlxsw_devlink_sb_port_pool_set,
1280 	.sb_tc_pool_bind_get		= mlxsw_devlink_sb_tc_pool_bind_get,
1281 	.sb_tc_pool_bind_set		= mlxsw_devlink_sb_tc_pool_bind_set,
1282 	.sb_occ_snapshot		= mlxsw_devlink_sb_occ_snapshot,
1283 	.sb_occ_max_clear		= mlxsw_devlink_sb_occ_max_clear,
1284 	.sb_occ_port_pool_get		= mlxsw_devlink_sb_occ_port_pool_get,
1285 	.sb_occ_tc_port_bind_get	= mlxsw_devlink_sb_occ_tc_port_bind_get,
1286 	.info_get			= mlxsw_devlink_info_get,
1287 	.flash_update			= mlxsw_devlink_flash_update,
1288 	.trap_init			= mlxsw_devlink_trap_init,
1289 	.trap_fini			= mlxsw_devlink_trap_fini,
1290 	.trap_action_set		= mlxsw_devlink_trap_action_set,
1291 	.trap_group_init		= mlxsw_devlink_trap_group_init,
1292 	.trap_group_set			= mlxsw_devlink_trap_group_set,
1293 	.trap_policer_init		= mlxsw_devlink_trap_policer_init,
1294 	.trap_policer_fini		= mlxsw_devlink_trap_policer_fini,
1295 	.trap_policer_set		= mlxsw_devlink_trap_policer_set,
1296 	.trap_policer_counter_get	= mlxsw_devlink_trap_policer_counter_get,
1297 };
1298 
1299 static int
1300 __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
1301 				 const struct mlxsw_bus *mlxsw_bus,
1302 				 void *bus_priv, bool reload,
1303 				 struct devlink *devlink,
1304 				 struct netlink_ext_ack *extack)
1305 {
1306 	const char *device_kind = mlxsw_bus_info->device_kind;
1307 	struct mlxsw_core *mlxsw_core;
1308 	struct mlxsw_driver *mlxsw_driver;
1309 	struct mlxsw_res *res;
1310 	size_t alloc_size;
1311 	int err;
1312 
1313 	mlxsw_driver = mlxsw_core_driver_get(device_kind);
1314 	if (!mlxsw_driver)
1315 		return -EINVAL;
1316 
1317 	if (!reload) {
1318 		alloc_size = sizeof(*mlxsw_core) + mlxsw_driver->priv_size;
1319 		devlink = devlink_alloc(&mlxsw_devlink_ops, alloc_size);
1320 		if (!devlink) {
1321 			err = -ENOMEM;
1322 			goto err_devlink_alloc;
1323 		}
1324 	}
1325 
1326 	mlxsw_core = devlink_priv(devlink);
1327 	INIT_LIST_HEAD(&mlxsw_core->rx_listener_list);
1328 	INIT_LIST_HEAD(&mlxsw_core->event_listener_list);
1329 	mlxsw_core->driver = mlxsw_driver;
1330 	mlxsw_core->bus = mlxsw_bus;
1331 	mlxsw_core->bus_priv = bus_priv;
1332 	mlxsw_core->bus_info = mlxsw_bus_info;
1333 
1334 	res = mlxsw_driver->res_query_enabled ? &mlxsw_core->res : NULL;
1335 	err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile, res);
1336 	if (err)
1337 		goto err_bus_init;
1338 
1339 	if (mlxsw_driver->resources_register && !reload) {
1340 		err = mlxsw_driver->resources_register(mlxsw_core);
1341 		if (err)
1342 			goto err_register_resources;
1343 	}
1344 
1345 	err = mlxsw_ports_init(mlxsw_core);
1346 	if (err)
1347 		goto err_ports_init;
1348 
1349 	if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) &&
1350 	    MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) {
1351 		alloc_size = sizeof(u8) *
1352 			MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) *
1353 			MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS);
1354 		mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL);
1355 		if (!mlxsw_core->lag.mapping) {
1356 			err = -ENOMEM;
1357 			goto err_alloc_lag_mapping;
1358 		}
1359 	}
1360 
1361 	err = mlxsw_emad_init(mlxsw_core);
1362 	if (err)
1363 		goto err_emad_init;
1364 
1365 	if (!reload) {
1366 		err = devlink_register(devlink, mlxsw_bus_info->dev);
1367 		if (err)
1368 			goto err_devlink_register;
1369 	}
1370 
1371 	if (mlxsw_driver->params_register && !reload) {
1372 		err = mlxsw_driver->params_register(mlxsw_core);
1373 		if (err)
1374 			goto err_register_params;
1375 	}
1376 
1377 	if (mlxsw_driver->init) {
1378 		err = mlxsw_driver->init(mlxsw_core, mlxsw_bus_info, extack);
1379 		if (err)
1380 			goto err_driver_init;
1381 	}
1382 
1383 	err = mlxsw_hwmon_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->hwmon);
1384 	if (err)
1385 		goto err_hwmon_init;
1386 
1387 	err = mlxsw_thermal_init(mlxsw_core, mlxsw_bus_info,
1388 				 &mlxsw_core->thermal);
1389 	if (err)
1390 		goto err_thermal_init;
1391 
1392 	if (mlxsw_driver->params_register)
1393 		devlink_params_publish(devlink);
1394 
1395 	if (!reload)
1396 		devlink_reload_enable(devlink);
1397 
1398 	return 0;
1399 
1400 err_thermal_init:
1401 	mlxsw_hwmon_fini(mlxsw_core->hwmon);
1402 err_hwmon_init:
1403 	if (mlxsw_core->driver->fini)
1404 		mlxsw_core->driver->fini(mlxsw_core);
1405 err_driver_init:
1406 	if (mlxsw_driver->params_unregister && !reload)
1407 		mlxsw_driver->params_unregister(mlxsw_core);
1408 err_register_params:
1409 	if (!reload)
1410 		devlink_unregister(devlink);
1411 err_devlink_register:
1412 	mlxsw_emad_fini(mlxsw_core);
1413 err_emad_init:
1414 	kfree(mlxsw_core->lag.mapping);
1415 err_alloc_lag_mapping:
1416 	mlxsw_ports_fini(mlxsw_core);
1417 err_ports_init:
1418 	if (!reload)
1419 		devlink_resources_unregister(devlink, NULL);
1420 err_register_resources:
1421 	mlxsw_bus->fini(bus_priv);
1422 err_bus_init:
1423 	if (!reload)
1424 		devlink_free(devlink);
1425 err_devlink_alloc:
1426 	return err;
1427 }
1428 
1429 int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
1430 				   const struct mlxsw_bus *mlxsw_bus,
1431 				   void *bus_priv, bool reload,
1432 				   struct devlink *devlink,
1433 				   struct netlink_ext_ack *extack)
1434 {
1435 	bool called_again = false;
1436 	int err;
1437 
1438 again:
1439 	err = __mlxsw_core_bus_device_register(mlxsw_bus_info, mlxsw_bus,
1440 					       bus_priv, reload,
1441 					       devlink, extack);
1442 	/* -EAGAIN is returned in case the FW was updated. FW needs
1443 	 * a reset, so lets try to call __mlxsw_core_bus_device_register()
1444 	 * again.
1445 	 */
1446 	if (err == -EAGAIN && !called_again) {
1447 		called_again = true;
1448 		goto again;
1449 	}
1450 
1451 	return err;
1452 }
1453 EXPORT_SYMBOL(mlxsw_core_bus_device_register);
1454 
1455 void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core,
1456 				      bool reload)
1457 {
1458 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
1459 
1460 	if (!reload)
1461 		devlink_reload_disable(devlink);
1462 	if (devlink_is_reload_failed(devlink)) {
1463 		if (!reload)
1464 			/* Only the parts that were not de-initialized in the
1465 			 * failed reload attempt need to be de-initialized.
1466 			 */
1467 			goto reload_fail_deinit;
1468 		else
1469 			return;
1470 	}
1471 
1472 	if (mlxsw_core->driver->params_unregister)
1473 		devlink_params_unpublish(devlink);
1474 	mlxsw_thermal_fini(mlxsw_core->thermal);
1475 	mlxsw_hwmon_fini(mlxsw_core->hwmon);
1476 	if (mlxsw_core->driver->fini)
1477 		mlxsw_core->driver->fini(mlxsw_core);
1478 	if (mlxsw_core->driver->params_unregister && !reload)
1479 		mlxsw_core->driver->params_unregister(mlxsw_core);
1480 	if (!reload)
1481 		devlink_unregister(devlink);
1482 	mlxsw_emad_fini(mlxsw_core);
1483 	kfree(mlxsw_core->lag.mapping);
1484 	mlxsw_ports_fini(mlxsw_core);
1485 	if (!reload)
1486 		devlink_resources_unregister(devlink, NULL);
1487 	mlxsw_core->bus->fini(mlxsw_core->bus_priv);
1488 
1489 	return;
1490 
1491 reload_fail_deinit:
1492 	if (mlxsw_core->driver->params_unregister)
1493 		mlxsw_core->driver->params_unregister(mlxsw_core);
1494 	devlink_unregister(devlink);
1495 	devlink_resources_unregister(devlink, NULL);
1496 	devlink_free(devlink);
1497 }
1498 EXPORT_SYMBOL(mlxsw_core_bus_device_unregister);
1499 
1500 bool mlxsw_core_skb_transmit_busy(struct mlxsw_core *mlxsw_core,
1501 				  const struct mlxsw_tx_info *tx_info)
1502 {
1503 	return mlxsw_core->bus->skb_transmit_busy(mlxsw_core->bus_priv,
1504 						  tx_info);
1505 }
1506 EXPORT_SYMBOL(mlxsw_core_skb_transmit_busy);
1507 
1508 int mlxsw_core_skb_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
1509 			    const struct mlxsw_tx_info *tx_info)
1510 {
1511 	return mlxsw_core->bus->skb_transmit(mlxsw_core->bus_priv, skb,
1512 					     tx_info);
1513 }
1514 EXPORT_SYMBOL(mlxsw_core_skb_transmit);
1515 
1516 void mlxsw_core_ptp_transmitted(struct mlxsw_core *mlxsw_core,
1517 				struct sk_buff *skb, u8 local_port)
1518 {
1519 	if (mlxsw_core->driver->ptp_transmitted)
1520 		mlxsw_core->driver->ptp_transmitted(mlxsw_core, skb,
1521 						    local_port);
1522 }
1523 EXPORT_SYMBOL(mlxsw_core_ptp_transmitted);
1524 
1525 static bool __is_rx_listener_equal(const struct mlxsw_rx_listener *rxl_a,
1526 				   const struct mlxsw_rx_listener *rxl_b)
1527 {
1528 	return (rxl_a->func == rxl_b->func &&
1529 		rxl_a->local_port == rxl_b->local_port &&
1530 		rxl_a->trap_id == rxl_b->trap_id &&
1531 		rxl_a->mirror_reason == rxl_b->mirror_reason);
1532 }
1533 
1534 static struct mlxsw_rx_listener_item *
1535 __find_rx_listener_item(struct mlxsw_core *mlxsw_core,
1536 			const struct mlxsw_rx_listener *rxl)
1537 {
1538 	struct mlxsw_rx_listener_item *rxl_item;
1539 
1540 	list_for_each_entry(rxl_item, &mlxsw_core->rx_listener_list, list) {
1541 		if (__is_rx_listener_equal(&rxl_item->rxl, rxl))
1542 			return rxl_item;
1543 	}
1544 	return NULL;
1545 }
1546 
1547 int mlxsw_core_rx_listener_register(struct mlxsw_core *mlxsw_core,
1548 				    const struct mlxsw_rx_listener *rxl,
1549 				    void *priv, bool enabled)
1550 {
1551 	struct mlxsw_rx_listener_item *rxl_item;
1552 
1553 	rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
1554 	if (rxl_item)
1555 		return -EEXIST;
1556 	rxl_item = kmalloc(sizeof(*rxl_item), GFP_KERNEL);
1557 	if (!rxl_item)
1558 		return -ENOMEM;
1559 	rxl_item->rxl = *rxl;
1560 	rxl_item->priv = priv;
1561 	rxl_item->enabled = enabled;
1562 
1563 	list_add_rcu(&rxl_item->list, &mlxsw_core->rx_listener_list);
1564 	return 0;
1565 }
1566 EXPORT_SYMBOL(mlxsw_core_rx_listener_register);
1567 
1568 void mlxsw_core_rx_listener_unregister(struct mlxsw_core *mlxsw_core,
1569 				       const struct mlxsw_rx_listener *rxl)
1570 {
1571 	struct mlxsw_rx_listener_item *rxl_item;
1572 
1573 	rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
1574 	if (!rxl_item)
1575 		return;
1576 	list_del_rcu(&rxl_item->list);
1577 	synchronize_rcu();
1578 	kfree(rxl_item);
1579 }
1580 EXPORT_SYMBOL(mlxsw_core_rx_listener_unregister);
1581 
1582 static void
1583 mlxsw_core_rx_listener_state_set(struct mlxsw_core *mlxsw_core,
1584 				 const struct mlxsw_rx_listener *rxl,
1585 				 bool enabled)
1586 {
1587 	struct mlxsw_rx_listener_item *rxl_item;
1588 
1589 	rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
1590 	if (WARN_ON(!rxl_item))
1591 		return;
1592 	rxl_item->enabled = enabled;
1593 }
1594 
1595 static void mlxsw_core_event_listener_func(struct sk_buff *skb, u8 local_port,
1596 					   void *priv)
1597 {
1598 	struct mlxsw_event_listener_item *event_listener_item = priv;
1599 	struct mlxsw_reg_info reg;
1600 	char *payload;
1601 	char *reg_tlv;
1602 	char *op_tlv;
1603 
1604 	mlxsw_emad_tlv_parse(skb);
1605 	op_tlv = mlxsw_emad_op_tlv(skb);
1606 	reg_tlv = mlxsw_emad_reg_tlv(skb);
1607 
1608 	reg.id = mlxsw_emad_op_tlv_register_id_get(op_tlv);
1609 	reg.len = (mlxsw_emad_reg_tlv_len_get(reg_tlv) - 1) * sizeof(u32);
1610 	payload = mlxsw_emad_reg_payload(reg_tlv);
1611 	event_listener_item->el.func(&reg, payload, event_listener_item->priv);
1612 	dev_kfree_skb(skb);
1613 }
1614 
1615 static bool __is_event_listener_equal(const struct mlxsw_event_listener *el_a,
1616 				      const struct mlxsw_event_listener *el_b)
1617 {
1618 	return (el_a->func == el_b->func &&
1619 		el_a->trap_id == el_b->trap_id);
1620 }
1621 
1622 static struct mlxsw_event_listener_item *
1623 __find_event_listener_item(struct mlxsw_core *mlxsw_core,
1624 			   const struct mlxsw_event_listener *el)
1625 {
1626 	struct mlxsw_event_listener_item *el_item;
1627 
1628 	list_for_each_entry(el_item, &mlxsw_core->event_listener_list, list) {
1629 		if (__is_event_listener_equal(&el_item->el, el))
1630 			return el_item;
1631 	}
1632 	return NULL;
1633 }
1634 
1635 int mlxsw_core_event_listener_register(struct mlxsw_core *mlxsw_core,
1636 				       const struct mlxsw_event_listener *el,
1637 				       void *priv)
1638 {
1639 	int err;
1640 	struct mlxsw_event_listener_item *el_item;
1641 	const struct mlxsw_rx_listener rxl = {
1642 		.func = mlxsw_core_event_listener_func,
1643 		.local_port = MLXSW_PORT_DONT_CARE,
1644 		.trap_id = el->trap_id,
1645 	};
1646 
1647 	el_item = __find_event_listener_item(mlxsw_core, el);
1648 	if (el_item)
1649 		return -EEXIST;
1650 	el_item = kmalloc(sizeof(*el_item), GFP_KERNEL);
1651 	if (!el_item)
1652 		return -ENOMEM;
1653 	el_item->el = *el;
1654 	el_item->priv = priv;
1655 
1656 	err = mlxsw_core_rx_listener_register(mlxsw_core, &rxl, el_item, true);
1657 	if (err)
1658 		goto err_rx_listener_register;
1659 
1660 	/* No reason to save item if we did not manage to register an RX
1661 	 * listener for it.
1662 	 */
1663 	list_add_rcu(&el_item->list, &mlxsw_core->event_listener_list);
1664 
1665 	return 0;
1666 
1667 err_rx_listener_register:
1668 	kfree(el_item);
1669 	return err;
1670 }
1671 EXPORT_SYMBOL(mlxsw_core_event_listener_register);
1672 
1673 void mlxsw_core_event_listener_unregister(struct mlxsw_core *mlxsw_core,
1674 					  const struct mlxsw_event_listener *el)
1675 {
1676 	struct mlxsw_event_listener_item *el_item;
1677 	const struct mlxsw_rx_listener rxl = {
1678 		.func = mlxsw_core_event_listener_func,
1679 		.local_port = MLXSW_PORT_DONT_CARE,
1680 		.trap_id = el->trap_id,
1681 	};
1682 
1683 	el_item = __find_event_listener_item(mlxsw_core, el);
1684 	if (!el_item)
1685 		return;
1686 	mlxsw_core_rx_listener_unregister(mlxsw_core, &rxl);
1687 	list_del(&el_item->list);
1688 	kfree(el_item);
1689 }
1690 EXPORT_SYMBOL(mlxsw_core_event_listener_unregister);
1691 
1692 static int mlxsw_core_listener_register(struct mlxsw_core *mlxsw_core,
1693 					const struct mlxsw_listener *listener,
1694 					void *priv, bool enabled)
1695 {
1696 	if (listener->is_event) {
1697 		WARN_ON(!enabled);
1698 		return mlxsw_core_event_listener_register(mlxsw_core,
1699 						&listener->event_listener,
1700 						priv);
1701 	} else {
1702 		return mlxsw_core_rx_listener_register(mlxsw_core,
1703 						&listener->rx_listener,
1704 						priv, enabled);
1705 	}
1706 }
1707 
1708 static void mlxsw_core_listener_unregister(struct mlxsw_core *mlxsw_core,
1709 				      const struct mlxsw_listener *listener,
1710 				      void *priv)
1711 {
1712 	if (listener->is_event)
1713 		mlxsw_core_event_listener_unregister(mlxsw_core,
1714 						     &listener->event_listener);
1715 	else
1716 		mlxsw_core_rx_listener_unregister(mlxsw_core,
1717 						  &listener->rx_listener);
1718 }
1719 
1720 int mlxsw_core_trap_register(struct mlxsw_core *mlxsw_core,
1721 			     const struct mlxsw_listener *listener, void *priv)
1722 {
1723 	enum mlxsw_reg_htgt_trap_group trap_group;
1724 	enum mlxsw_reg_hpkt_action action;
1725 	char hpkt_pl[MLXSW_REG_HPKT_LEN];
1726 	int err;
1727 
1728 	err = mlxsw_core_listener_register(mlxsw_core, listener, priv,
1729 					   listener->enabled_on_register);
1730 	if (err)
1731 		return err;
1732 
1733 	action = listener->enabled_on_register ? listener->en_action :
1734 						 listener->dis_action;
1735 	trap_group = listener->enabled_on_register ? listener->en_trap_group :
1736 						     listener->dis_trap_group;
1737 	mlxsw_reg_hpkt_pack(hpkt_pl, action, listener->trap_id,
1738 			    trap_group, listener->is_ctrl);
1739 	err = mlxsw_reg_write(mlxsw_core,  MLXSW_REG(hpkt), hpkt_pl);
1740 	if (err)
1741 		goto err_trap_set;
1742 
1743 	return 0;
1744 
1745 err_trap_set:
1746 	mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
1747 	return err;
1748 }
1749 EXPORT_SYMBOL(mlxsw_core_trap_register);
1750 
1751 void mlxsw_core_trap_unregister(struct mlxsw_core *mlxsw_core,
1752 				const struct mlxsw_listener *listener,
1753 				void *priv)
1754 {
1755 	char hpkt_pl[MLXSW_REG_HPKT_LEN];
1756 
1757 	if (!listener->is_event) {
1758 		mlxsw_reg_hpkt_pack(hpkt_pl, listener->dis_action,
1759 				    listener->trap_id, listener->dis_trap_group,
1760 				    listener->is_ctrl);
1761 		mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
1762 	}
1763 
1764 	mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
1765 }
1766 EXPORT_SYMBOL(mlxsw_core_trap_unregister);
1767 
1768 int mlxsw_core_trap_state_set(struct mlxsw_core *mlxsw_core,
1769 			      const struct mlxsw_listener *listener,
1770 			      bool enabled)
1771 {
1772 	enum mlxsw_reg_htgt_trap_group trap_group;
1773 	enum mlxsw_reg_hpkt_action action;
1774 	char hpkt_pl[MLXSW_REG_HPKT_LEN];
1775 	int err;
1776 
1777 	/* Not supported for event listener */
1778 	if (WARN_ON(listener->is_event))
1779 		return -EINVAL;
1780 
1781 	action = enabled ? listener->en_action : listener->dis_action;
1782 	trap_group = enabled ? listener->en_trap_group :
1783 			       listener->dis_trap_group;
1784 	mlxsw_reg_hpkt_pack(hpkt_pl, action, listener->trap_id,
1785 			    trap_group, listener->is_ctrl);
1786 	err = mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
1787 	if (err)
1788 		return err;
1789 
1790 	mlxsw_core_rx_listener_state_set(mlxsw_core, &listener->rx_listener,
1791 					 enabled);
1792 	return 0;
1793 }
1794 EXPORT_SYMBOL(mlxsw_core_trap_state_set);
1795 
1796 static u64 mlxsw_core_tid_get(struct mlxsw_core *mlxsw_core)
1797 {
1798 	return atomic64_inc_return(&mlxsw_core->emad.tid);
1799 }
1800 
1801 static int mlxsw_core_reg_access_emad(struct mlxsw_core *mlxsw_core,
1802 				      const struct mlxsw_reg_info *reg,
1803 				      char *payload,
1804 				      enum mlxsw_core_reg_access_type type,
1805 				      struct list_head *bulk_list,
1806 				      mlxsw_reg_trans_cb_t *cb,
1807 				      unsigned long cb_priv)
1808 {
1809 	u64 tid = mlxsw_core_tid_get(mlxsw_core);
1810 	struct mlxsw_reg_trans *trans;
1811 	int err;
1812 
1813 	trans = kzalloc(sizeof(*trans), GFP_KERNEL);
1814 	if (!trans)
1815 		return -ENOMEM;
1816 
1817 	err = mlxsw_emad_reg_access(mlxsw_core, reg, payload, type, trans,
1818 				    bulk_list, cb, cb_priv, tid);
1819 	if (err) {
1820 		kfree_rcu(trans, rcu);
1821 		return err;
1822 	}
1823 	return 0;
1824 }
1825 
1826 int mlxsw_reg_trans_query(struct mlxsw_core *mlxsw_core,
1827 			  const struct mlxsw_reg_info *reg, char *payload,
1828 			  struct list_head *bulk_list,
1829 			  mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
1830 {
1831 	return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
1832 					  MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
1833 					  bulk_list, cb, cb_priv);
1834 }
1835 EXPORT_SYMBOL(mlxsw_reg_trans_query);
1836 
1837 int mlxsw_reg_trans_write(struct mlxsw_core *mlxsw_core,
1838 			  const struct mlxsw_reg_info *reg, char *payload,
1839 			  struct list_head *bulk_list,
1840 			  mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
1841 {
1842 	return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
1843 					  MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
1844 					  bulk_list, cb, cb_priv);
1845 }
1846 EXPORT_SYMBOL(mlxsw_reg_trans_write);
1847 
1848 #define MLXSW_REG_TRANS_ERR_STRING_SIZE	256
1849 
1850 static int mlxsw_reg_trans_wait(struct mlxsw_reg_trans *trans)
1851 {
1852 	char err_string[MLXSW_REG_TRANS_ERR_STRING_SIZE];
1853 	struct mlxsw_core *mlxsw_core = trans->core;
1854 	int err;
1855 
1856 	wait_for_completion(&trans->completion);
1857 	cancel_delayed_work_sync(&trans->timeout_dw);
1858 	err = trans->err;
1859 
1860 	if (trans->retries)
1861 		dev_warn(mlxsw_core->bus_info->dev, "EMAD retries (%d/%d) (tid=%llx)\n",
1862 			 trans->retries, MLXSW_EMAD_MAX_RETRY, trans->tid);
1863 	if (err) {
1864 		dev_err(mlxsw_core->bus_info->dev, "EMAD reg access failed (tid=%llx,reg_id=%x(%s),type=%s,status=%x(%s))\n",
1865 			trans->tid, trans->reg->id,
1866 			mlxsw_reg_id_str(trans->reg->id),
1867 			mlxsw_core_reg_access_type_str(trans->type),
1868 			trans->emad_status,
1869 			mlxsw_emad_op_tlv_status_str(trans->emad_status));
1870 
1871 		snprintf(err_string, MLXSW_REG_TRANS_ERR_STRING_SIZE,
1872 			 "(tid=%llx,reg_id=%x(%s)) %s (%s)\n", trans->tid,
1873 			 trans->reg->id, mlxsw_reg_id_str(trans->reg->id),
1874 			 mlxsw_emad_op_tlv_status_str(trans->emad_status),
1875 			 trans->emad_err_string ? trans->emad_err_string : "");
1876 
1877 		trace_devlink_hwerr(priv_to_devlink(mlxsw_core),
1878 				    trans->emad_status, err_string);
1879 
1880 		kfree(trans->emad_err_string);
1881 	}
1882 
1883 	list_del(&trans->bulk_list);
1884 	kfree_rcu(trans, rcu);
1885 	return err;
1886 }
1887 
1888 int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list)
1889 {
1890 	struct mlxsw_reg_trans *trans;
1891 	struct mlxsw_reg_trans *tmp;
1892 	int sum_err = 0;
1893 	int err;
1894 
1895 	list_for_each_entry_safe(trans, tmp, bulk_list, bulk_list) {
1896 		err = mlxsw_reg_trans_wait(trans);
1897 		if (err && sum_err == 0)
1898 			sum_err = err; /* first error to be returned */
1899 	}
1900 	return sum_err;
1901 }
1902 EXPORT_SYMBOL(mlxsw_reg_trans_bulk_wait);
1903 
1904 static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
1905 				     const struct mlxsw_reg_info *reg,
1906 				     char *payload,
1907 				     enum mlxsw_core_reg_access_type type)
1908 {
1909 	enum mlxsw_emad_op_tlv_status status;
1910 	int err, n_retry;
1911 	bool reset_ok;
1912 	char *in_mbox, *out_mbox, *tmp;
1913 
1914 	dev_dbg(mlxsw_core->bus_info->dev, "Reg cmd access (reg_id=%x(%s),type=%s)\n",
1915 		reg->id, mlxsw_reg_id_str(reg->id),
1916 		mlxsw_core_reg_access_type_str(type));
1917 
1918 	in_mbox = mlxsw_cmd_mbox_alloc();
1919 	if (!in_mbox)
1920 		return -ENOMEM;
1921 
1922 	out_mbox = mlxsw_cmd_mbox_alloc();
1923 	if (!out_mbox) {
1924 		err = -ENOMEM;
1925 		goto free_in_mbox;
1926 	}
1927 
1928 	mlxsw_emad_pack_op_tlv(in_mbox, reg, type,
1929 			       mlxsw_core_tid_get(mlxsw_core));
1930 	tmp = in_mbox + MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
1931 	mlxsw_emad_pack_reg_tlv(tmp, reg, payload);
1932 
1933 	/* There is a special treatment needed for MRSR (reset) register.
1934 	 * The command interface will return error after the command
1935 	 * is executed, so tell the lower layer to expect it
1936 	 * and cope accordingly.
1937 	 */
1938 	reset_ok = reg->id == MLXSW_REG_MRSR_ID;
1939 
1940 	n_retry = 0;
1941 retry:
1942 	err = mlxsw_cmd_access_reg(mlxsw_core, reset_ok, in_mbox, out_mbox);
1943 	if (!err) {
1944 		err = mlxsw_emad_process_status(out_mbox, &status);
1945 		if (err) {
1946 			if (err == -EAGAIN && n_retry++ < MLXSW_EMAD_MAX_RETRY)
1947 				goto retry;
1948 			dev_err(mlxsw_core->bus_info->dev, "Reg cmd access status failed (status=%x(%s))\n",
1949 				status, mlxsw_emad_op_tlv_status_str(status));
1950 		}
1951 	}
1952 
1953 	if (!err)
1954 		memcpy(payload, mlxsw_emad_reg_payload_cmd(out_mbox),
1955 		       reg->len);
1956 
1957 	mlxsw_cmd_mbox_free(out_mbox);
1958 free_in_mbox:
1959 	mlxsw_cmd_mbox_free(in_mbox);
1960 	if (err)
1961 		dev_err(mlxsw_core->bus_info->dev, "Reg cmd access failed (reg_id=%x(%s),type=%s)\n",
1962 			reg->id, mlxsw_reg_id_str(reg->id),
1963 			mlxsw_core_reg_access_type_str(type));
1964 	return err;
1965 }
1966 
1967 static void mlxsw_core_reg_access_cb(struct mlxsw_core *mlxsw_core,
1968 				     char *payload, size_t payload_len,
1969 				     unsigned long cb_priv)
1970 {
1971 	char *orig_payload = (char *) cb_priv;
1972 
1973 	memcpy(orig_payload, payload, payload_len);
1974 }
1975 
1976 static int mlxsw_core_reg_access(struct mlxsw_core *mlxsw_core,
1977 				 const struct mlxsw_reg_info *reg,
1978 				 char *payload,
1979 				 enum mlxsw_core_reg_access_type type)
1980 {
1981 	LIST_HEAD(bulk_list);
1982 	int err;
1983 
1984 	/* During initialization EMAD interface is not available to us,
1985 	 * so we default to command interface. We switch to EMAD interface
1986 	 * after setting the appropriate traps.
1987 	 */
1988 	if (!mlxsw_core->emad.use_emad)
1989 		return mlxsw_core_reg_access_cmd(mlxsw_core, reg,
1990 						 payload, type);
1991 
1992 	err = mlxsw_core_reg_access_emad(mlxsw_core, reg,
1993 					 payload, type, &bulk_list,
1994 					 mlxsw_core_reg_access_cb,
1995 					 (unsigned long) payload);
1996 	if (err)
1997 		return err;
1998 	return mlxsw_reg_trans_bulk_wait(&bulk_list);
1999 }
2000 
2001 int mlxsw_reg_query(struct mlxsw_core *mlxsw_core,
2002 		    const struct mlxsw_reg_info *reg, char *payload)
2003 {
2004 	return mlxsw_core_reg_access(mlxsw_core, reg, payload,
2005 				     MLXSW_CORE_REG_ACCESS_TYPE_QUERY);
2006 }
2007 EXPORT_SYMBOL(mlxsw_reg_query);
2008 
2009 int mlxsw_reg_write(struct mlxsw_core *mlxsw_core,
2010 		    const struct mlxsw_reg_info *reg, char *payload)
2011 {
2012 	return mlxsw_core_reg_access(mlxsw_core, reg, payload,
2013 				     MLXSW_CORE_REG_ACCESS_TYPE_WRITE);
2014 }
2015 EXPORT_SYMBOL(mlxsw_reg_write);
2016 
2017 void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
2018 			    struct mlxsw_rx_info *rx_info)
2019 {
2020 	struct mlxsw_rx_listener_item *rxl_item;
2021 	const struct mlxsw_rx_listener *rxl;
2022 	u8 local_port;
2023 	bool found = false;
2024 
2025 	if (rx_info->is_lag) {
2026 		dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: lag_id = %d, lag_port_index = 0x%x\n",
2027 				    __func__, rx_info->u.lag_id,
2028 				    rx_info->trap_id);
2029 		/* Upper layer does not care if the skb came from LAG or not,
2030 		 * so just get the local_port for the lag port and push it up.
2031 		 */
2032 		local_port = mlxsw_core_lag_mapping_get(mlxsw_core,
2033 							rx_info->u.lag_id,
2034 							rx_info->lag_port_index);
2035 	} else {
2036 		local_port = rx_info->u.sys_port;
2037 	}
2038 
2039 	dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: local_port = %d, trap_id = 0x%x\n",
2040 			    __func__, local_port, rx_info->trap_id);
2041 
2042 	if ((rx_info->trap_id >= MLXSW_TRAP_ID_MAX) ||
2043 	    (local_port >= mlxsw_core->max_ports))
2044 		goto drop;
2045 
2046 	rcu_read_lock();
2047 	list_for_each_entry_rcu(rxl_item, &mlxsw_core->rx_listener_list, list) {
2048 		rxl = &rxl_item->rxl;
2049 		if ((rxl->local_port == MLXSW_PORT_DONT_CARE ||
2050 		     rxl->local_port == local_port) &&
2051 		    rxl->trap_id == rx_info->trap_id &&
2052 		    rxl->mirror_reason == rx_info->mirror_reason) {
2053 			if (rxl_item->enabled)
2054 				found = true;
2055 			break;
2056 		}
2057 	}
2058 	if (!found) {
2059 		rcu_read_unlock();
2060 		goto drop;
2061 	}
2062 
2063 	rxl->func(skb, local_port, rxl_item->priv);
2064 	rcu_read_unlock();
2065 	return;
2066 
2067 drop:
2068 	dev_kfree_skb(skb);
2069 }
2070 EXPORT_SYMBOL(mlxsw_core_skb_receive);
2071 
2072 static int mlxsw_core_lag_mapping_index(struct mlxsw_core *mlxsw_core,
2073 					u16 lag_id, u8 port_index)
2074 {
2075 	return MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS) * lag_id +
2076 	       port_index;
2077 }
2078 
2079 void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core,
2080 				u16 lag_id, u8 port_index, u8 local_port)
2081 {
2082 	int index = mlxsw_core_lag_mapping_index(mlxsw_core,
2083 						 lag_id, port_index);
2084 
2085 	mlxsw_core->lag.mapping[index] = local_port;
2086 }
2087 EXPORT_SYMBOL(mlxsw_core_lag_mapping_set);
2088 
2089 u8 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core,
2090 			      u16 lag_id, u8 port_index)
2091 {
2092 	int index = mlxsw_core_lag_mapping_index(mlxsw_core,
2093 						 lag_id, port_index);
2094 
2095 	return mlxsw_core->lag.mapping[index];
2096 }
2097 EXPORT_SYMBOL(mlxsw_core_lag_mapping_get);
2098 
2099 void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core,
2100 				  u16 lag_id, u8 local_port)
2101 {
2102 	int i;
2103 
2104 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); i++) {
2105 		int index = mlxsw_core_lag_mapping_index(mlxsw_core,
2106 							 lag_id, i);
2107 
2108 		if (mlxsw_core->lag.mapping[index] == local_port)
2109 			mlxsw_core->lag.mapping[index] = 0;
2110 	}
2111 }
2112 EXPORT_SYMBOL(mlxsw_core_lag_mapping_clear);
2113 
2114 bool mlxsw_core_res_valid(struct mlxsw_core *mlxsw_core,
2115 			  enum mlxsw_res_id res_id)
2116 {
2117 	return mlxsw_res_valid(&mlxsw_core->res, res_id);
2118 }
2119 EXPORT_SYMBOL(mlxsw_core_res_valid);
2120 
2121 u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core,
2122 		       enum mlxsw_res_id res_id)
2123 {
2124 	return mlxsw_res_get(&mlxsw_core->res, res_id);
2125 }
2126 EXPORT_SYMBOL(mlxsw_core_res_get);
2127 
2128 static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port,
2129 				  enum devlink_port_flavour flavour,
2130 				  u32 port_number, bool split,
2131 				  u32 split_port_subnumber,
2132 				  bool splittable, u32 lanes,
2133 				  const unsigned char *switch_id,
2134 				  unsigned char switch_id_len)
2135 {
2136 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
2137 	struct mlxsw_core_port *mlxsw_core_port =
2138 					&mlxsw_core->ports[local_port];
2139 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2140 	struct devlink_port_attrs attrs = {};
2141 	int err;
2142 
2143 	attrs.split = split;
2144 	attrs.lanes = lanes;
2145 	attrs.splittable = splittable;
2146 	attrs.flavour = flavour;
2147 	attrs.phys.port_number = port_number;
2148 	attrs.phys.split_subport_number = split_port_subnumber;
2149 	memcpy(attrs.switch_id.id, switch_id, switch_id_len);
2150 	attrs.switch_id.id_len = switch_id_len;
2151 	mlxsw_core_port->local_port = local_port;
2152 	devlink_port_attrs_set(devlink_port, &attrs);
2153 	err = devlink_port_register(devlink, devlink_port, local_port);
2154 	if (err)
2155 		memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
2156 	return err;
2157 }
2158 
2159 static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
2160 {
2161 	struct mlxsw_core_port *mlxsw_core_port =
2162 					&mlxsw_core->ports[local_port];
2163 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2164 
2165 	devlink_port_unregister(devlink_port);
2166 	memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
2167 }
2168 
2169 int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port,
2170 			 u32 port_number, bool split,
2171 			 u32 split_port_subnumber,
2172 			 bool splittable, u32 lanes,
2173 			 const unsigned char *switch_id,
2174 			 unsigned char switch_id_len)
2175 {
2176 	return __mlxsw_core_port_init(mlxsw_core, local_port,
2177 				      DEVLINK_PORT_FLAVOUR_PHYSICAL,
2178 				      port_number, split, split_port_subnumber,
2179 				      splittable, lanes,
2180 				      switch_id, switch_id_len);
2181 }
2182 EXPORT_SYMBOL(mlxsw_core_port_init);
2183 
2184 void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
2185 {
2186 	__mlxsw_core_port_fini(mlxsw_core, local_port);
2187 }
2188 EXPORT_SYMBOL(mlxsw_core_port_fini);
2189 
2190 int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core,
2191 			     void *port_driver_priv,
2192 			     const unsigned char *switch_id,
2193 			     unsigned char switch_id_len)
2194 {
2195 	struct mlxsw_core_port *mlxsw_core_port =
2196 				&mlxsw_core->ports[MLXSW_PORT_CPU_PORT];
2197 	int err;
2198 
2199 	err = __mlxsw_core_port_init(mlxsw_core, MLXSW_PORT_CPU_PORT,
2200 				     DEVLINK_PORT_FLAVOUR_CPU,
2201 				     0, false, 0, false, 0,
2202 				     switch_id, switch_id_len);
2203 	if (err)
2204 		return err;
2205 
2206 	mlxsw_core_port->port_driver_priv = port_driver_priv;
2207 	return 0;
2208 }
2209 EXPORT_SYMBOL(mlxsw_core_cpu_port_init);
2210 
2211 void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core)
2212 {
2213 	__mlxsw_core_port_fini(mlxsw_core, MLXSW_PORT_CPU_PORT);
2214 }
2215 EXPORT_SYMBOL(mlxsw_core_cpu_port_fini);
2216 
2217 void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port,
2218 			     void *port_driver_priv, struct net_device *dev)
2219 {
2220 	struct mlxsw_core_port *mlxsw_core_port =
2221 					&mlxsw_core->ports[local_port];
2222 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2223 
2224 	mlxsw_core_port->port_driver_priv = port_driver_priv;
2225 	devlink_port_type_eth_set(devlink_port, dev);
2226 }
2227 EXPORT_SYMBOL(mlxsw_core_port_eth_set);
2228 
2229 void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port,
2230 			    void *port_driver_priv)
2231 {
2232 	struct mlxsw_core_port *mlxsw_core_port =
2233 					&mlxsw_core->ports[local_port];
2234 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2235 
2236 	mlxsw_core_port->port_driver_priv = port_driver_priv;
2237 	devlink_port_type_ib_set(devlink_port, NULL);
2238 }
2239 EXPORT_SYMBOL(mlxsw_core_port_ib_set);
2240 
2241 void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port,
2242 			   void *port_driver_priv)
2243 {
2244 	struct mlxsw_core_port *mlxsw_core_port =
2245 					&mlxsw_core->ports[local_port];
2246 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2247 
2248 	mlxsw_core_port->port_driver_priv = port_driver_priv;
2249 	devlink_port_type_clear(devlink_port);
2250 }
2251 EXPORT_SYMBOL(mlxsw_core_port_clear);
2252 
2253 enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
2254 						u8 local_port)
2255 {
2256 	struct mlxsw_core_port *mlxsw_core_port =
2257 					&mlxsw_core->ports[local_port];
2258 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2259 
2260 	return devlink_port->type;
2261 }
2262 EXPORT_SYMBOL(mlxsw_core_port_type_get);
2263 
2264 
2265 struct devlink_port *
2266 mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core,
2267 				 u8 local_port)
2268 {
2269 	struct mlxsw_core_port *mlxsw_core_port =
2270 					&mlxsw_core->ports[local_port];
2271 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2272 
2273 	return devlink_port;
2274 }
2275 EXPORT_SYMBOL(mlxsw_core_port_devlink_port_get);
2276 
2277 int mlxsw_core_module_max_width(struct mlxsw_core *mlxsw_core, u8 module)
2278 {
2279 	enum mlxsw_reg_pmtm_module_type module_type;
2280 	char pmtm_pl[MLXSW_REG_PMTM_LEN];
2281 	int err;
2282 
2283 	mlxsw_reg_pmtm_pack(pmtm_pl, module);
2284 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(pmtm), pmtm_pl);
2285 	if (err)
2286 		return err;
2287 	mlxsw_reg_pmtm_unpack(pmtm_pl, &module_type);
2288 
2289 	/* Here we need to get the module width according to the module type. */
2290 
2291 	switch (module_type) {
2292 	case MLXSW_REG_PMTM_MODULE_TYPE_C2C8X: /* fall through */
2293 	case MLXSW_REG_PMTM_MODULE_TYPE_QSFP_DD: /* fall through */
2294 	case MLXSW_REG_PMTM_MODULE_TYPE_OSFP:
2295 		return 8;
2296 	case MLXSW_REG_PMTM_MODULE_TYPE_C2C4X: /* fall through */
2297 	case MLXSW_REG_PMTM_MODULE_TYPE_BP_4X: /* fall through */
2298 	case MLXSW_REG_PMTM_MODULE_TYPE_QSFP:
2299 		return 4;
2300 	case MLXSW_REG_PMTM_MODULE_TYPE_C2C2X: /* fall through */
2301 	case MLXSW_REG_PMTM_MODULE_TYPE_BP_2X: /* fall through */
2302 	case MLXSW_REG_PMTM_MODULE_TYPE_SFP_DD: /* fall through */
2303 	case MLXSW_REG_PMTM_MODULE_TYPE_DSFP:
2304 		return 2;
2305 	case MLXSW_REG_PMTM_MODULE_TYPE_C2C1X: /* fall through */
2306 	case MLXSW_REG_PMTM_MODULE_TYPE_BP_1X: /* fall through */
2307 	case MLXSW_REG_PMTM_MODULE_TYPE_SFP:
2308 		return 1;
2309 	default:
2310 		return -EINVAL;
2311 	}
2312 }
2313 EXPORT_SYMBOL(mlxsw_core_module_max_width);
2314 
2315 static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core,
2316 				    const char *buf, size_t size)
2317 {
2318 	__be32 *m = (__be32 *) buf;
2319 	int i;
2320 	int count = size / sizeof(__be32);
2321 
2322 	for (i = count - 1; i >= 0; i--)
2323 		if (m[i])
2324 			break;
2325 	i++;
2326 	count = i ? i : 1;
2327 	for (i = 0; i < count; i += 4)
2328 		dev_dbg(mlxsw_core->bus_info->dev, "%04x - %08x %08x %08x %08x\n",
2329 			i * 4, be32_to_cpu(m[i]), be32_to_cpu(m[i + 1]),
2330 			be32_to_cpu(m[i + 2]), be32_to_cpu(m[i + 3]));
2331 }
2332 
2333 int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
2334 		   u32 in_mod, bool out_mbox_direct, bool reset_ok,
2335 		   char *in_mbox, size_t in_mbox_size,
2336 		   char *out_mbox, size_t out_mbox_size)
2337 {
2338 	u8 status;
2339 	int err;
2340 
2341 	BUG_ON(in_mbox_size % sizeof(u32) || out_mbox_size % sizeof(u32));
2342 	if (!mlxsw_core->bus->cmd_exec)
2343 		return -EOPNOTSUPP;
2344 
2345 	dev_dbg(mlxsw_core->bus_info->dev, "Cmd exec (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
2346 		opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod, in_mod);
2347 	if (in_mbox) {
2348 		dev_dbg(mlxsw_core->bus_info->dev, "Input mailbox:\n");
2349 		mlxsw_core_buf_dump_dbg(mlxsw_core, in_mbox, in_mbox_size);
2350 	}
2351 
2352 	err = mlxsw_core->bus->cmd_exec(mlxsw_core->bus_priv, opcode,
2353 					opcode_mod, in_mod, out_mbox_direct,
2354 					in_mbox, in_mbox_size,
2355 					out_mbox, out_mbox_size, &status);
2356 
2357 	if (!err && out_mbox) {
2358 		dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n");
2359 		mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size);
2360 	}
2361 
2362 	if (reset_ok && err == -EIO &&
2363 	    status == MLXSW_CMD_STATUS_RUNNING_RESET) {
2364 		err = 0;
2365 	} else if (err == -EIO && status != MLXSW_CMD_STATUS_OK) {
2366 		dev_err(mlxsw_core->bus_info->dev, "Cmd exec failed (opcode=%x(%s),opcode_mod=%x,in_mod=%x,status=%x(%s))\n",
2367 			opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
2368 			in_mod, status, mlxsw_cmd_status_str(status));
2369 	} else if (err == -ETIMEDOUT) {
2370 		dev_err(mlxsw_core->bus_info->dev, "Cmd exec timed-out (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
2371 			opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
2372 			in_mod);
2373 	}
2374 
2375 	return err;
2376 }
2377 EXPORT_SYMBOL(mlxsw_cmd_exec);
2378 
2379 int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay)
2380 {
2381 	return queue_delayed_work(mlxsw_wq, dwork, delay);
2382 }
2383 EXPORT_SYMBOL(mlxsw_core_schedule_dw);
2384 
2385 bool mlxsw_core_schedule_work(struct work_struct *work)
2386 {
2387 	return queue_work(mlxsw_owq, work);
2388 }
2389 EXPORT_SYMBOL(mlxsw_core_schedule_work);
2390 
2391 void mlxsw_core_flush_owq(void)
2392 {
2393 	flush_workqueue(mlxsw_owq);
2394 }
2395 EXPORT_SYMBOL(mlxsw_core_flush_owq);
2396 
2397 int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core,
2398 			     const struct mlxsw_config_profile *profile,
2399 			     u64 *p_single_size, u64 *p_double_size,
2400 			     u64 *p_linear_size)
2401 {
2402 	struct mlxsw_driver *driver = mlxsw_core->driver;
2403 
2404 	if (!driver->kvd_sizes_get)
2405 		return -EINVAL;
2406 
2407 	return driver->kvd_sizes_get(mlxsw_core, profile,
2408 				     p_single_size, p_double_size,
2409 				     p_linear_size);
2410 }
2411 EXPORT_SYMBOL(mlxsw_core_kvd_sizes_get);
2412 
2413 void mlxsw_core_fw_flash_start(struct mlxsw_core *mlxsw_core)
2414 {
2415 	mlxsw_core->fw_flash_in_progress = true;
2416 }
2417 EXPORT_SYMBOL(mlxsw_core_fw_flash_start);
2418 
2419 void mlxsw_core_fw_flash_end(struct mlxsw_core *mlxsw_core)
2420 {
2421 	mlxsw_core->fw_flash_in_progress = false;
2422 }
2423 EXPORT_SYMBOL(mlxsw_core_fw_flash_end);
2424 
2425 int mlxsw_core_resources_query(struct mlxsw_core *mlxsw_core, char *mbox,
2426 			       struct mlxsw_res *res)
2427 {
2428 	int index, i;
2429 	u64 data;
2430 	u16 id;
2431 	int err;
2432 
2433 	if (!res)
2434 		return 0;
2435 
2436 	mlxsw_cmd_mbox_zero(mbox);
2437 
2438 	for (index = 0; index < MLXSW_CMD_QUERY_RESOURCES_MAX_QUERIES;
2439 	     index++) {
2440 		err = mlxsw_cmd_query_resources(mlxsw_core, mbox, index);
2441 		if (err)
2442 			return err;
2443 
2444 		for (i = 0; i < MLXSW_CMD_QUERY_RESOURCES_PER_QUERY; i++) {
2445 			id = mlxsw_cmd_mbox_query_resource_id_get(mbox, i);
2446 			data = mlxsw_cmd_mbox_query_resource_data_get(mbox, i);
2447 
2448 			if (id == MLXSW_CMD_QUERY_RESOURCES_TABLE_END_ID)
2449 				return 0;
2450 
2451 			mlxsw_res_parse(res, id, data);
2452 		}
2453 	}
2454 
2455 	/* If after MLXSW_RESOURCES_QUERY_MAX_QUERIES we still didn't get
2456 	 * MLXSW_RESOURCES_TABLE_END_ID, something went bad in the FW.
2457 	 */
2458 	return -EIO;
2459 }
2460 EXPORT_SYMBOL(mlxsw_core_resources_query);
2461 
2462 u32 mlxsw_core_read_frc_h(struct mlxsw_core *mlxsw_core)
2463 {
2464 	return mlxsw_core->bus->read_frc_h(mlxsw_core->bus_priv);
2465 }
2466 EXPORT_SYMBOL(mlxsw_core_read_frc_h);
2467 
2468 u32 mlxsw_core_read_frc_l(struct mlxsw_core *mlxsw_core)
2469 {
2470 	return mlxsw_core->bus->read_frc_l(mlxsw_core->bus_priv);
2471 }
2472 EXPORT_SYMBOL(mlxsw_core_read_frc_l);
2473 
2474 void mlxsw_core_emad_string_tlv_enable(struct mlxsw_core *mlxsw_core)
2475 {
2476 	mlxsw_core->emad.enable_string_tlv = true;
2477 }
2478 EXPORT_SYMBOL(mlxsw_core_emad_string_tlv_enable);
2479 
2480 static int __init mlxsw_core_module_init(void)
2481 {
2482 	int err;
2483 
2484 	mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, 0, 0);
2485 	if (!mlxsw_wq)
2486 		return -ENOMEM;
2487 	mlxsw_owq = alloc_ordered_workqueue("%s_ordered", 0,
2488 					    mlxsw_core_driver_name);
2489 	if (!mlxsw_owq) {
2490 		err = -ENOMEM;
2491 		goto err_alloc_ordered_workqueue;
2492 	}
2493 	return 0;
2494 
2495 err_alloc_ordered_workqueue:
2496 	destroy_workqueue(mlxsw_wq);
2497 	return err;
2498 }
2499 
2500 static void __exit mlxsw_core_module_exit(void)
2501 {
2502 	destroy_workqueue(mlxsw_owq);
2503 	destroy_workqueue(mlxsw_wq);
2504 }
2505 
2506 module_init(mlxsw_core_module_init);
2507 module_exit(mlxsw_core_module_exit);
2508 
2509 MODULE_LICENSE("Dual BSD/GPL");
2510 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
2511 MODULE_DESCRIPTION("Mellanox switch device core driver");
2512