1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/device.h>
7 #include <linux/export.h>
8 #include <linux/err.h>
9 #include <linux/if_link.h>
10 #include <linux/netdevice.h>
11 #include <linux/completion.h>
12 #include <linux/skbuff.h>
13 #include <linux/etherdevice.h>
14 #include <linux/types.h>
15 #include <linux/string.h>
16 #include <linux/gfp.h>
17 #include <linux/random.h>
18 #include <linux/jiffies.h>
19 #include <linux/mutex.h>
20 #include <linux/rcupdate.h>
21 #include <linux/slab.h>
22 #include <linux/workqueue.h>
23 #include <linux/firmware.h>
24 #include <asm/byteorder.h>
25 #include <net/devlink.h>
26 #include <trace/events/devlink.h>
27 
28 #include "core.h"
29 #include "core_env.h"
30 #include "item.h"
31 #include "cmd.h"
32 #include "port.h"
33 #include "trap.h"
34 #include "emad.h"
35 #include "reg.h"
36 #include "resources.h"
37 #include "../mlxfw/mlxfw.h"
38 
39 static LIST_HEAD(mlxsw_core_driver_list);
40 static DEFINE_SPINLOCK(mlxsw_core_driver_list_lock);
41 
42 static const char mlxsw_core_driver_name[] = "mlxsw_core";
43 
44 static struct workqueue_struct *mlxsw_wq;
45 static struct workqueue_struct *mlxsw_owq;
46 
47 struct mlxsw_core_port {
48 	struct devlink_port devlink_port;
49 	void *port_driver_priv;
50 	u16 local_port;
51 	struct mlxsw_linecard *linecard;
52 };
53 
54 void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port)
55 {
56 	return mlxsw_core_port->port_driver_priv;
57 }
58 EXPORT_SYMBOL(mlxsw_core_port_driver_priv);
59 
60 static bool mlxsw_core_port_check(struct mlxsw_core_port *mlxsw_core_port)
61 {
62 	return mlxsw_core_port->port_driver_priv != NULL;
63 }
64 
65 struct mlxsw_core {
66 	struct mlxsw_driver *driver;
67 	const struct mlxsw_bus *bus;
68 	void *bus_priv;
69 	const struct mlxsw_bus_info *bus_info;
70 	struct workqueue_struct *emad_wq;
71 	struct list_head rx_listener_list;
72 	struct list_head event_listener_list;
73 	struct list_head irq_event_handler_list;
74 	struct mutex irq_event_handler_lock; /* Locks access to handlers list */
75 	struct {
76 		atomic64_t tid;
77 		struct list_head trans_list;
78 		spinlock_t trans_list_lock; /* protects trans_list writes */
79 		bool use_emad;
80 		bool enable_string_tlv;
81 		bool enable_latency_tlv;
82 	} emad;
83 	struct {
84 		u16 *mapping; /* lag_id+port_index to local_port mapping */
85 	} lag;
86 	struct mlxsw_res res;
87 	struct mlxsw_hwmon *hwmon;
88 	struct mlxsw_thermal *thermal;
89 	struct mlxsw_linecards *linecards;
90 	struct mlxsw_core_port *ports;
91 	unsigned int max_ports;
92 	atomic_t active_ports_count;
93 	bool fw_flash_in_progress;
94 	struct {
95 		struct devlink_health_reporter *fw_fatal;
96 	} health;
97 	struct mlxsw_env *env;
98 	unsigned long driver_priv[];
99 	/* driver_priv has to be always the last item */
100 };
101 
102 struct mlxsw_linecards *mlxsw_core_linecards(struct mlxsw_core *mlxsw_core)
103 {
104 	return mlxsw_core->linecards;
105 }
106 
107 void mlxsw_core_linecards_set(struct mlxsw_core *mlxsw_core,
108 			      struct mlxsw_linecards *linecards)
109 {
110 	mlxsw_core->linecards = linecards;
111 }
112 
113 #define MLXSW_PORT_MAX_PORTS_DEFAULT	0x40
114 
115 static u64 mlxsw_ports_occ_get(void *priv)
116 {
117 	struct mlxsw_core *mlxsw_core = priv;
118 
119 	return atomic_read(&mlxsw_core->active_ports_count);
120 }
121 
122 static int mlxsw_core_resources_ports_register(struct mlxsw_core *mlxsw_core)
123 {
124 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
125 	struct devlink_resource_size_params ports_num_params;
126 	u32 max_ports;
127 
128 	max_ports = mlxsw_core->max_ports - 1;
129 	devlink_resource_size_params_init(&ports_num_params, max_ports,
130 					  max_ports, 1,
131 					  DEVLINK_RESOURCE_UNIT_ENTRY);
132 
133 	return devl_resource_register(devlink,
134 				      DEVLINK_RESOURCE_GENERIC_NAME_PORTS,
135 				      max_ports, MLXSW_CORE_RESOURCE_PORTS,
136 				      DEVLINK_RESOURCE_ID_PARENT_TOP,
137 				      &ports_num_params);
138 }
139 
140 static int mlxsw_ports_init(struct mlxsw_core *mlxsw_core, bool reload)
141 {
142 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
143 	int err;
144 
145 	/* Switch ports are numbered from 1 to queried value */
146 	if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_SYSTEM_PORT))
147 		mlxsw_core->max_ports = MLXSW_CORE_RES_GET(mlxsw_core,
148 							   MAX_SYSTEM_PORT) + 1;
149 	else
150 		mlxsw_core->max_ports = MLXSW_PORT_MAX_PORTS_DEFAULT + 1;
151 
152 	mlxsw_core->ports = kcalloc(mlxsw_core->max_ports,
153 				    sizeof(struct mlxsw_core_port), GFP_KERNEL);
154 	if (!mlxsw_core->ports)
155 		return -ENOMEM;
156 
157 	if (!reload) {
158 		err = mlxsw_core_resources_ports_register(mlxsw_core);
159 		if (err)
160 			goto err_resources_ports_register;
161 	}
162 	atomic_set(&mlxsw_core->active_ports_count, 0);
163 	devl_resource_occ_get_register(devlink, MLXSW_CORE_RESOURCE_PORTS,
164 				       mlxsw_ports_occ_get, mlxsw_core);
165 
166 	return 0;
167 
168 err_resources_ports_register:
169 	kfree(mlxsw_core->ports);
170 	return err;
171 }
172 
173 static void mlxsw_ports_fini(struct mlxsw_core *mlxsw_core, bool reload)
174 {
175 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
176 
177 	devl_resource_occ_get_unregister(devlink, MLXSW_CORE_RESOURCE_PORTS);
178 	if (!reload)
179 		devl_resources_unregister(priv_to_devlink(mlxsw_core));
180 
181 	kfree(mlxsw_core->ports);
182 }
183 
184 unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core)
185 {
186 	return mlxsw_core->max_ports;
187 }
188 EXPORT_SYMBOL(mlxsw_core_max_ports);
189 
190 int mlxsw_core_max_lag(struct mlxsw_core *mlxsw_core, u16 *p_max_lag)
191 {
192 	struct mlxsw_driver *driver = mlxsw_core->driver;
193 
194 	if (driver->profile->used_max_lag) {
195 		*p_max_lag = driver->profile->max_lag;
196 		return 0;
197 	}
198 
199 	if (!MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG))
200 		return -EIO;
201 
202 	*p_max_lag = MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG);
203 	return 0;
204 }
205 EXPORT_SYMBOL(mlxsw_core_max_lag);
206 
207 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core)
208 {
209 	return mlxsw_core->driver_priv;
210 }
211 EXPORT_SYMBOL(mlxsw_core_driver_priv);
212 
213 bool
214 mlxsw_core_fw_rev_minor_subminor_validate(const struct mlxsw_fw_rev *rev,
215 					  const struct mlxsw_fw_rev *req_rev)
216 {
217 	return rev->minor > req_rev->minor ||
218 	       (rev->minor == req_rev->minor &&
219 		rev->subminor >= req_rev->subminor);
220 }
221 EXPORT_SYMBOL(mlxsw_core_fw_rev_minor_subminor_validate);
222 
223 struct mlxsw_rx_listener_item {
224 	struct list_head list;
225 	struct mlxsw_rx_listener rxl;
226 	void *priv;
227 	bool enabled;
228 };
229 
230 struct mlxsw_event_listener_item {
231 	struct list_head list;
232 	struct mlxsw_core *mlxsw_core;
233 	struct mlxsw_event_listener el;
234 	void *priv;
235 };
236 
237 static const u8 mlxsw_core_trap_groups[] = {
238 	MLXSW_REG_HTGT_TRAP_GROUP_EMAD,
239 	MLXSW_REG_HTGT_TRAP_GROUP_CORE_EVENT,
240 };
241 
242 static int mlxsw_core_trap_groups_set(struct mlxsw_core *mlxsw_core)
243 {
244 	char htgt_pl[MLXSW_REG_HTGT_LEN];
245 	int err;
246 	int i;
247 
248 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
249 		return 0;
250 
251 	for (i = 0; i < ARRAY_SIZE(mlxsw_core_trap_groups); i++) {
252 		mlxsw_reg_htgt_pack(htgt_pl, mlxsw_core_trap_groups[i],
253 				    MLXSW_REG_HTGT_INVALID_POLICER,
254 				    MLXSW_REG_HTGT_DEFAULT_PRIORITY,
255 				    MLXSW_REG_HTGT_DEFAULT_TC);
256 		err = mlxsw_reg_write(mlxsw_core, MLXSW_REG(htgt), htgt_pl);
257 		if (err)
258 			return err;
259 	}
260 	return 0;
261 }
262 
263 /******************
264  * EMAD processing
265  ******************/
266 
267 /* emad_eth_hdr_dmac
268  * Destination MAC in EMAD's Ethernet header.
269  * Must be set to 01:02:c9:00:00:01
270  */
271 MLXSW_ITEM_BUF(emad, eth_hdr, dmac, 0x00, 6);
272 
273 /* emad_eth_hdr_smac
274  * Source MAC in EMAD's Ethernet header.
275  * Must be set to 00:02:c9:01:02:03
276  */
277 MLXSW_ITEM_BUF(emad, eth_hdr, smac, 0x06, 6);
278 
279 /* emad_eth_hdr_ethertype
280  * Ethertype in EMAD's Ethernet header.
281  * Must be set to 0x8932
282  */
283 MLXSW_ITEM32(emad, eth_hdr, ethertype, 0x0C, 16, 16);
284 
285 /* emad_eth_hdr_mlx_proto
286  * Mellanox protocol.
287  * Must be set to 0x0.
288  */
289 MLXSW_ITEM32(emad, eth_hdr, mlx_proto, 0x0C, 8, 8);
290 
291 /* emad_eth_hdr_ver
292  * Mellanox protocol version.
293  * Must be set to 0x0.
294  */
295 MLXSW_ITEM32(emad, eth_hdr, ver, 0x0C, 4, 4);
296 
297 /* emad_op_tlv_type
298  * Type of the TLV.
299  * Must be set to 0x1 (operation TLV).
300  */
301 MLXSW_ITEM32(emad, op_tlv, type, 0x00, 27, 5);
302 
303 /* emad_op_tlv_len
304  * Length of the operation TLV in u32.
305  * Must be set to 0x4.
306  */
307 MLXSW_ITEM32(emad, op_tlv, len, 0x00, 16, 11);
308 
309 /* emad_op_tlv_dr
310  * Direct route bit. Setting to 1 indicates the EMAD is a direct route
311  * EMAD. DR TLV must follow.
312  *
313  * Note: Currently not supported and must not be set.
314  */
315 MLXSW_ITEM32(emad, op_tlv, dr, 0x00, 15, 1);
316 
317 /* emad_op_tlv_status
318  * Returned status in case of EMAD response. Must be set to 0 in case
319  * of EMAD request.
320  * 0x0 - success
321  * 0x1 - device is busy. Requester should retry
322  * 0x2 - Mellanox protocol version not supported
323  * 0x3 - unknown TLV
324  * 0x4 - register not supported
325  * 0x5 - operation class not supported
326  * 0x6 - EMAD method not supported
327  * 0x7 - bad parameter (e.g. port out of range)
328  * 0x8 - resource not available
329  * 0x9 - message receipt acknowledgment. Requester should retry
330  * 0x70 - internal error
331  */
332 MLXSW_ITEM32(emad, op_tlv, status, 0x00, 8, 7);
333 
334 /* emad_op_tlv_register_id
335  * Register ID of register within register TLV.
336  */
337 MLXSW_ITEM32(emad, op_tlv, register_id, 0x04, 16, 16);
338 
339 /* emad_op_tlv_r
340  * Response bit. Setting to 1 indicates Response, otherwise request.
341  */
342 MLXSW_ITEM32(emad, op_tlv, r, 0x04, 15, 1);
343 
344 /* emad_op_tlv_method
345  * EMAD method type.
346  * 0x1 - query
347  * 0x2 - write
348  * 0x3 - send (currently not supported)
349  * 0x4 - event
350  */
351 MLXSW_ITEM32(emad, op_tlv, method, 0x04, 8, 7);
352 
353 /* emad_op_tlv_class
354  * EMAD operation class. Must be set to 0x1 (REG_ACCESS).
355  */
356 MLXSW_ITEM32(emad, op_tlv, class, 0x04, 0, 8);
357 
358 /* emad_op_tlv_tid
359  * EMAD transaction ID. Used for pairing request and response EMADs.
360  */
361 MLXSW_ITEM64(emad, op_tlv, tid, 0x08, 0, 64);
362 
363 /* emad_string_tlv_type
364  * Type of the TLV.
365  * Must be set to 0x2 (string TLV).
366  */
367 MLXSW_ITEM32(emad, string_tlv, type, 0x00, 27, 5);
368 
369 /* emad_string_tlv_len
370  * Length of the string TLV in u32.
371  */
372 MLXSW_ITEM32(emad, string_tlv, len, 0x00, 16, 11);
373 
374 #define MLXSW_EMAD_STRING_TLV_STRING_LEN 128
375 
376 /* emad_string_tlv_string
377  * String provided by the device's firmware in case of erroneous register access
378  */
379 MLXSW_ITEM_BUF(emad, string_tlv, string, 0x04,
380 	       MLXSW_EMAD_STRING_TLV_STRING_LEN);
381 
382 /* emad_latency_tlv_type
383  * Type of the TLV.
384  * Must be set to 0x4 (latency TLV).
385  */
386 MLXSW_ITEM32(emad, latency_tlv, type, 0x00, 27, 5);
387 
388 /* emad_latency_tlv_len
389  * Length of the latency TLV in u32.
390  */
391 MLXSW_ITEM32(emad, latency_tlv, len, 0x00, 16, 11);
392 
393 /* emad_latency_tlv_latency_time
394  * EMAD latency time in units of uSec.
395  */
396 MLXSW_ITEM32(emad, latency_tlv, latency_time, 0x04, 0, 32);
397 
398 /* emad_reg_tlv_type
399  * Type of the TLV.
400  * Must be set to 0x3 (register TLV).
401  */
402 MLXSW_ITEM32(emad, reg_tlv, type, 0x00, 27, 5);
403 
404 /* emad_reg_tlv_len
405  * Length of the operation TLV in u32.
406  */
407 MLXSW_ITEM32(emad, reg_tlv, len, 0x00, 16, 11);
408 
409 /* emad_end_tlv_type
410  * Type of the TLV.
411  * Must be set to 0x0 (end TLV).
412  */
413 MLXSW_ITEM32(emad, end_tlv, type, 0x00, 27, 5);
414 
415 /* emad_end_tlv_len
416  * Length of the end TLV in u32.
417  * Must be set to 1.
418  */
419 MLXSW_ITEM32(emad, end_tlv, len, 0x00, 16, 11);
420 
421 enum mlxsw_core_reg_access_type {
422 	MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
423 	MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
424 };
425 
426 static inline const char *
427 mlxsw_core_reg_access_type_str(enum mlxsw_core_reg_access_type type)
428 {
429 	switch (type) {
430 	case MLXSW_CORE_REG_ACCESS_TYPE_QUERY:
431 		return "query";
432 	case MLXSW_CORE_REG_ACCESS_TYPE_WRITE:
433 		return "write";
434 	}
435 	BUG();
436 }
437 
438 static void mlxsw_emad_pack_end_tlv(char *end_tlv)
439 {
440 	mlxsw_emad_end_tlv_type_set(end_tlv, MLXSW_EMAD_TLV_TYPE_END);
441 	mlxsw_emad_end_tlv_len_set(end_tlv, MLXSW_EMAD_END_TLV_LEN);
442 }
443 
444 static void mlxsw_emad_pack_reg_tlv(char *reg_tlv,
445 				    const struct mlxsw_reg_info *reg,
446 				    char *payload)
447 {
448 	mlxsw_emad_reg_tlv_type_set(reg_tlv, MLXSW_EMAD_TLV_TYPE_REG);
449 	mlxsw_emad_reg_tlv_len_set(reg_tlv, reg->len / sizeof(u32) + 1);
450 	memcpy(reg_tlv + sizeof(u32), payload, reg->len);
451 }
452 
453 static void mlxsw_emad_pack_string_tlv(char *string_tlv)
454 {
455 	mlxsw_emad_string_tlv_type_set(string_tlv, MLXSW_EMAD_TLV_TYPE_STRING);
456 	mlxsw_emad_string_tlv_len_set(string_tlv, MLXSW_EMAD_STRING_TLV_LEN);
457 }
458 
459 static void mlxsw_emad_pack_op_tlv(char *op_tlv,
460 				   const struct mlxsw_reg_info *reg,
461 				   enum mlxsw_core_reg_access_type type,
462 				   u64 tid)
463 {
464 	mlxsw_emad_op_tlv_type_set(op_tlv, MLXSW_EMAD_TLV_TYPE_OP);
465 	mlxsw_emad_op_tlv_len_set(op_tlv, MLXSW_EMAD_OP_TLV_LEN);
466 	mlxsw_emad_op_tlv_dr_set(op_tlv, 0);
467 	mlxsw_emad_op_tlv_status_set(op_tlv, 0);
468 	mlxsw_emad_op_tlv_register_id_set(op_tlv, reg->id);
469 	mlxsw_emad_op_tlv_r_set(op_tlv, MLXSW_EMAD_OP_TLV_REQUEST);
470 	if (type == MLXSW_CORE_REG_ACCESS_TYPE_QUERY)
471 		mlxsw_emad_op_tlv_method_set(op_tlv,
472 					     MLXSW_EMAD_OP_TLV_METHOD_QUERY);
473 	else
474 		mlxsw_emad_op_tlv_method_set(op_tlv,
475 					     MLXSW_EMAD_OP_TLV_METHOD_WRITE);
476 	mlxsw_emad_op_tlv_class_set(op_tlv,
477 				    MLXSW_EMAD_OP_TLV_CLASS_REG_ACCESS);
478 	mlxsw_emad_op_tlv_tid_set(op_tlv, tid);
479 }
480 
481 static void mlxsw_emad_pack_latency_tlv(char *latency_tlv)
482 {
483 	mlxsw_emad_latency_tlv_type_set(latency_tlv, MLXSW_EMAD_TLV_TYPE_LATENCY);
484 	mlxsw_emad_latency_tlv_len_set(latency_tlv, MLXSW_EMAD_LATENCY_TLV_LEN);
485 }
486 
487 static int mlxsw_emad_construct_eth_hdr(struct sk_buff *skb)
488 {
489 	char *eth_hdr = skb_push(skb, MLXSW_EMAD_ETH_HDR_LEN);
490 
491 	mlxsw_emad_eth_hdr_dmac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_DMAC);
492 	mlxsw_emad_eth_hdr_smac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_SMAC);
493 	mlxsw_emad_eth_hdr_ethertype_set(eth_hdr, MLXSW_EMAD_EH_ETHERTYPE);
494 	mlxsw_emad_eth_hdr_mlx_proto_set(eth_hdr, MLXSW_EMAD_EH_MLX_PROTO);
495 	mlxsw_emad_eth_hdr_ver_set(eth_hdr, MLXSW_EMAD_EH_PROTO_VERSION);
496 
497 	skb_reset_mac_header(skb);
498 
499 	return 0;
500 }
501 
502 static void mlxsw_emad_construct(const struct mlxsw_core *mlxsw_core,
503 				 struct sk_buff *skb,
504 				 const struct mlxsw_reg_info *reg,
505 				 char *payload,
506 				 enum mlxsw_core_reg_access_type type, u64 tid)
507 {
508 	char *buf;
509 
510 	buf = skb_push(skb, MLXSW_EMAD_END_TLV_LEN * sizeof(u32));
511 	mlxsw_emad_pack_end_tlv(buf);
512 
513 	buf = skb_push(skb, reg->len + sizeof(u32));
514 	mlxsw_emad_pack_reg_tlv(buf, reg, payload);
515 
516 	if (mlxsw_core->emad.enable_latency_tlv) {
517 		buf = skb_push(skb, MLXSW_EMAD_LATENCY_TLV_LEN * sizeof(u32));
518 		mlxsw_emad_pack_latency_tlv(buf);
519 	}
520 
521 	if (mlxsw_core->emad.enable_string_tlv) {
522 		buf = skb_push(skb, MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32));
523 		mlxsw_emad_pack_string_tlv(buf);
524 	}
525 
526 	buf = skb_push(skb, MLXSW_EMAD_OP_TLV_LEN * sizeof(u32));
527 	mlxsw_emad_pack_op_tlv(buf, reg, type, tid);
528 
529 	mlxsw_emad_construct_eth_hdr(skb);
530 }
531 
532 struct mlxsw_emad_tlv_offsets {
533 	u16 op_tlv;
534 	u16 string_tlv;
535 	u16 latency_tlv;
536 	u16 reg_tlv;
537 };
538 
539 static bool mlxsw_emad_tlv_is_string_tlv(const char *tlv)
540 {
541 	u8 tlv_type = mlxsw_emad_string_tlv_type_get(tlv);
542 
543 	return tlv_type == MLXSW_EMAD_TLV_TYPE_STRING;
544 }
545 
546 static bool mlxsw_emad_tlv_is_latency_tlv(const char *tlv)
547 {
548 	u8 tlv_type = mlxsw_emad_latency_tlv_type_get(tlv);
549 
550 	return tlv_type == MLXSW_EMAD_TLV_TYPE_LATENCY;
551 }
552 
553 static void mlxsw_emad_tlv_parse(struct sk_buff *skb)
554 {
555 	struct mlxsw_emad_tlv_offsets *offsets =
556 		(struct mlxsw_emad_tlv_offsets *) skb->cb;
557 
558 	offsets->op_tlv = MLXSW_EMAD_ETH_HDR_LEN;
559 	offsets->string_tlv = 0;
560 	offsets->latency_tlv = 0;
561 
562 	offsets->reg_tlv = MLXSW_EMAD_ETH_HDR_LEN +
563 			   MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
564 
565 	/* If string TLV is present, it must come after the operation TLV. */
566 	if (mlxsw_emad_tlv_is_string_tlv(skb->data + offsets->reg_tlv)) {
567 		offsets->string_tlv = offsets->reg_tlv;
568 		offsets->reg_tlv += MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32);
569 	}
570 
571 	if (mlxsw_emad_tlv_is_latency_tlv(skb->data + offsets->reg_tlv)) {
572 		offsets->latency_tlv = offsets->reg_tlv;
573 		offsets->reg_tlv += MLXSW_EMAD_LATENCY_TLV_LEN * sizeof(u32);
574 	}
575 }
576 
577 static char *mlxsw_emad_op_tlv(const struct sk_buff *skb)
578 {
579 	struct mlxsw_emad_tlv_offsets *offsets =
580 		(struct mlxsw_emad_tlv_offsets *) skb->cb;
581 
582 	return ((char *) (skb->data + offsets->op_tlv));
583 }
584 
585 static char *mlxsw_emad_string_tlv(const struct sk_buff *skb)
586 {
587 	struct mlxsw_emad_tlv_offsets *offsets =
588 		(struct mlxsw_emad_tlv_offsets *) skb->cb;
589 
590 	if (!offsets->string_tlv)
591 		return NULL;
592 
593 	return ((char *) (skb->data + offsets->string_tlv));
594 }
595 
596 static char *mlxsw_emad_reg_tlv(const struct sk_buff *skb)
597 {
598 	struct mlxsw_emad_tlv_offsets *offsets =
599 		(struct mlxsw_emad_tlv_offsets *) skb->cb;
600 
601 	return ((char *) (skb->data + offsets->reg_tlv));
602 }
603 
604 static char *mlxsw_emad_reg_payload(const char *reg_tlv)
605 {
606 	return ((char *) (reg_tlv + sizeof(u32)));
607 }
608 
609 static char *mlxsw_emad_reg_payload_cmd(const char *mbox)
610 {
611 	return ((char *) (mbox + (MLXSW_EMAD_OP_TLV_LEN + 1) * sizeof(u32)));
612 }
613 
614 static u64 mlxsw_emad_get_tid(const struct sk_buff *skb)
615 {
616 	char *op_tlv;
617 
618 	op_tlv = mlxsw_emad_op_tlv(skb);
619 	return mlxsw_emad_op_tlv_tid_get(op_tlv);
620 }
621 
622 static bool mlxsw_emad_is_resp(const struct sk_buff *skb)
623 {
624 	char *op_tlv;
625 
626 	op_tlv = mlxsw_emad_op_tlv(skb);
627 	return (mlxsw_emad_op_tlv_r_get(op_tlv) == MLXSW_EMAD_OP_TLV_RESPONSE);
628 }
629 
630 static int mlxsw_emad_process_status(char *op_tlv,
631 				     enum mlxsw_emad_op_tlv_status *p_status)
632 {
633 	*p_status = mlxsw_emad_op_tlv_status_get(op_tlv);
634 
635 	switch (*p_status) {
636 	case MLXSW_EMAD_OP_TLV_STATUS_SUCCESS:
637 		return 0;
638 	case MLXSW_EMAD_OP_TLV_STATUS_BUSY:
639 	case MLXSW_EMAD_OP_TLV_STATUS_MESSAGE_RECEIPT_ACK:
640 		return -EAGAIN;
641 	case MLXSW_EMAD_OP_TLV_STATUS_VERSION_NOT_SUPPORTED:
642 	case MLXSW_EMAD_OP_TLV_STATUS_UNKNOWN_TLV:
643 	case MLXSW_EMAD_OP_TLV_STATUS_REGISTER_NOT_SUPPORTED:
644 	case MLXSW_EMAD_OP_TLV_STATUS_CLASS_NOT_SUPPORTED:
645 	case MLXSW_EMAD_OP_TLV_STATUS_METHOD_NOT_SUPPORTED:
646 	case MLXSW_EMAD_OP_TLV_STATUS_BAD_PARAMETER:
647 	case MLXSW_EMAD_OP_TLV_STATUS_RESOURCE_NOT_AVAILABLE:
648 	case MLXSW_EMAD_OP_TLV_STATUS_INTERNAL_ERROR:
649 	default:
650 		return -EIO;
651 	}
652 }
653 
654 static int
655 mlxsw_emad_process_status_skb(struct sk_buff *skb,
656 			      enum mlxsw_emad_op_tlv_status *p_status)
657 {
658 	return mlxsw_emad_process_status(mlxsw_emad_op_tlv(skb), p_status);
659 }
660 
661 struct mlxsw_reg_trans {
662 	struct list_head list;
663 	struct list_head bulk_list;
664 	struct mlxsw_core *core;
665 	struct sk_buff *tx_skb;
666 	struct mlxsw_tx_info tx_info;
667 	struct delayed_work timeout_dw;
668 	unsigned int retries;
669 	u64 tid;
670 	struct completion completion;
671 	atomic_t active;
672 	mlxsw_reg_trans_cb_t *cb;
673 	unsigned long cb_priv;
674 	const struct mlxsw_reg_info *reg;
675 	enum mlxsw_core_reg_access_type type;
676 	int err;
677 	char *emad_err_string;
678 	enum mlxsw_emad_op_tlv_status emad_status;
679 	struct rcu_head rcu;
680 };
681 
682 static void mlxsw_emad_process_string_tlv(const struct sk_buff *skb,
683 					  struct mlxsw_reg_trans *trans)
684 {
685 	char *string_tlv;
686 	char *string;
687 
688 	string_tlv = mlxsw_emad_string_tlv(skb);
689 	if (!string_tlv)
690 		return;
691 
692 	trans->emad_err_string = kzalloc(MLXSW_EMAD_STRING_TLV_STRING_LEN,
693 					 GFP_ATOMIC);
694 	if (!trans->emad_err_string)
695 		return;
696 
697 	string = mlxsw_emad_string_tlv_string_data(string_tlv);
698 	strscpy(trans->emad_err_string, string,
699 		MLXSW_EMAD_STRING_TLV_STRING_LEN);
700 }
701 
702 #define MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS	3000
703 #define MLXSW_EMAD_TIMEOUT_MS			200
704 
705 static void mlxsw_emad_trans_timeout_schedule(struct mlxsw_reg_trans *trans)
706 {
707 	unsigned long timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_MS);
708 
709 	if (trans->core->fw_flash_in_progress)
710 		timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS);
711 
712 	queue_delayed_work(trans->core->emad_wq, &trans->timeout_dw,
713 			   timeout << trans->retries);
714 }
715 
716 static int mlxsw_emad_transmit(struct mlxsw_core *mlxsw_core,
717 			       struct mlxsw_reg_trans *trans)
718 {
719 	struct sk_buff *skb;
720 	int err;
721 
722 	skb = skb_clone(trans->tx_skb, GFP_KERNEL);
723 	if (!skb)
724 		return -ENOMEM;
725 
726 	trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), false, 0,
727 			    skb->data + mlxsw_core->driver->txhdr_len,
728 			    skb->len - mlxsw_core->driver->txhdr_len);
729 
730 	atomic_set(&trans->active, 1);
731 	err = mlxsw_core_skb_transmit(mlxsw_core, skb, &trans->tx_info);
732 	if (err) {
733 		dev_kfree_skb(skb);
734 		return err;
735 	}
736 	mlxsw_emad_trans_timeout_schedule(trans);
737 	return 0;
738 }
739 
740 static void mlxsw_emad_trans_finish(struct mlxsw_reg_trans *trans, int err)
741 {
742 	struct mlxsw_core *mlxsw_core = trans->core;
743 
744 	dev_kfree_skb(trans->tx_skb);
745 	spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
746 	list_del_rcu(&trans->list);
747 	spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
748 	trans->err = err;
749 	complete(&trans->completion);
750 }
751 
752 static void mlxsw_emad_transmit_retry(struct mlxsw_core *mlxsw_core,
753 				      struct mlxsw_reg_trans *trans)
754 {
755 	int err;
756 
757 	if (trans->retries < MLXSW_EMAD_MAX_RETRY) {
758 		trans->retries++;
759 		err = mlxsw_emad_transmit(trans->core, trans);
760 		if (err == 0)
761 			return;
762 
763 		if (!atomic_dec_and_test(&trans->active))
764 			return;
765 	} else {
766 		err = -EIO;
767 	}
768 	mlxsw_emad_trans_finish(trans, err);
769 }
770 
771 static void mlxsw_emad_trans_timeout_work(struct work_struct *work)
772 {
773 	struct mlxsw_reg_trans *trans = container_of(work,
774 						     struct mlxsw_reg_trans,
775 						     timeout_dw.work);
776 
777 	if (!atomic_dec_and_test(&trans->active))
778 		return;
779 
780 	mlxsw_emad_transmit_retry(trans->core, trans);
781 }
782 
783 static void mlxsw_emad_process_response(struct mlxsw_core *mlxsw_core,
784 					struct mlxsw_reg_trans *trans,
785 					struct sk_buff *skb)
786 {
787 	int err;
788 
789 	if (!atomic_dec_and_test(&trans->active))
790 		return;
791 
792 	err = mlxsw_emad_process_status_skb(skb, &trans->emad_status);
793 	if (err == -EAGAIN) {
794 		mlxsw_emad_transmit_retry(mlxsw_core, trans);
795 	} else {
796 		if (err == 0) {
797 			char *reg_tlv = mlxsw_emad_reg_tlv(skb);
798 
799 			if (trans->cb)
800 				trans->cb(mlxsw_core,
801 					  mlxsw_emad_reg_payload(reg_tlv),
802 					  trans->reg->len, trans->cb_priv);
803 		} else {
804 			mlxsw_emad_process_string_tlv(skb, trans);
805 		}
806 		mlxsw_emad_trans_finish(trans, err);
807 	}
808 }
809 
810 /* called with rcu read lock held */
811 static void mlxsw_emad_rx_listener_func(struct sk_buff *skb, u16 local_port,
812 					void *priv)
813 {
814 	struct mlxsw_core *mlxsw_core = priv;
815 	struct mlxsw_reg_trans *trans;
816 
817 	trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), true, 0,
818 			    skb->data, skb->len);
819 
820 	mlxsw_emad_tlv_parse(skb);
821 
822 	if (!mlxsw_emad_is_resp(skb))
823 		goto free_skb;
824 
825 	list_for_each_entry_rcu(trans, &mlxsw_core->emad.trans_list, list) {
826 		if (mlxsw_emad_get_tid(skb) == trans->tid) {
827 			mlxsw_emad_process_response(mlxsw_core, trans, skb);
828 			break;
829 		}
830 	}
831 
832 free_skb:
833 	dev_kfree_skb(skb);
834 }
835 
836 static const struct mlxsw_listener mlxsw_emad_rx_listener =
837 	MLXSW_RXL(mlxsw_emad_rx_listener_func, ETHEMAD, TRAP_TO_CPU, false,
838 		  EMAD, FORWARD);
839 
840 static int mlxsw_emad_tlv_enable(struct mlxsw_core *mlxsw_core)
841 {
842 	char mgir_pl[MLXSW_REG_MGIR_LEN];
843 	bool string_tlv, latency_tlv;
844 	int err;
845 
846 	mlxsw_reg_mgir_pack(mgir_pl);
847 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mgir), mgir_pl);
848 	if (err)
849 		return err;
850 
851 	string_tlv = mlxsw_reg_mgir_fw_info_string_tlv_get(mgir_pl);
852 	mlxsw_core->emad.enable_string_tlv = string_tlv;
853 
854 	latency_tlv = mlxsw_reg_mgir_fw_info_latency_tlv_get(mgir_pl);
855 	mlxsw_core->emad.enable_latency_tlv = latency_tlv;
856 
857 	return 0;
858 }
859 
860 static void mlxsw_emad_tlv_disable(struct mlxsw_core *mlxsw_core)
861 {
862 	mlxsw_core->emad.enable_latency_tlv = false;
863 	mlxsw_core->emad.enable_string_tlv = false;
864 }
865 
866 static int mlxsw_emad_init(struct mlxsw_core *mlxsw_core)
867 {
868 	struct workqueue_struct *emad_wq;
869 	u64 tid;
870 	int err;
871 
872 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
873 		return 0;
874 
875 	emad_wq = alloc_workqueue("mlxsw_core_emad", 0, 0);
876 	if (!emad_wq)
877 		return -ENOMEM;
878 	mlxsw_core->emad_wq = emad_wq;
879 
880 	/* Set the upper 32 bits of the transaction ID field to a random
881 	 * number. This allows us to discard EMADs addressed to other
882 	 * devices.
883 	 */
884 	get_random_bytes(&tid, 4);
885 	tid <<= 32;
886 	atomic64_set(&mlxsw_core->emad.tid, tid);
887 
888 	INIT_LIST_HEAD(&mlxsw_core->emad.trans_list);
889 	spin_lock_init(&mlxsw_core->emad.trans_list_lock);
890 
891 	err = mlxsw_core_trap_register(mlxsw_core, &mlxsw_emad_rx_listener,
892 				       mlxsw_core);
893 	if (err)
894 		goto err_trap_register;
895 
896 	err = mlxsw_emad_tlv_enable(mlxsw_core);
897 	if (err)
898 		goto err_emad_tlv_enable;
899 
900 	mlxsw_core->emad.use_emad = true;
901 
902 	return 0;
903 
904 err_emad_tlv_enable:
905 	mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
906 				   mlxsw_core);
907 err_trap_register:
908 	destroy_workqueue(mlxsw_core->emad_wq);
909 	return err;
910 }
911 
912 static void mlxsw_emad_fini(struct mlxsw_core *mlxsw_core)
913 {
914 
915 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
916 		return;
917 
918 	mlxsw_core->emad.use_emad = false;
919 	mlxsw_emad_tlv_disable(mlxsw_core);
920 	mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
921 				   mlxsw_core);
922 	destroy_workqueue(mlxsw_core->emad_wq);
923 }
924 
925 static struct sk_buff *mlxsw_emad_alloc(const struct mlxsw_core *mlxsw_core,
926 					u16 reg_len)
927 {
928 	struct sk_buff *skb;
929 	u16 emad_len;
930 
931 	emad_len = (reg_len + sizeof(u32) + MLXSW_EMAD_ETH_HDR_LEN +
932 		    (MLXSW_EMAD_OP_TLV_LEN + MLXSW_EMAD_END_TLV_LEN) *
933 		    sizeof(u32) + mlxsw_core->driver->txhdr_len);
934 	if (mlxsw_core->emad.enable_string_tlv)
935 		emad_len += MLXSW_EMAD_STRING_TLV_LEN * sizeof(u32);
936 	if (mlxsw_core->emad.enable_latency_tlv)
937 		emad_len +=  MLXSW_EMAD_LATENCY_TLV_LEN * sizeof(u32);
938 	if (emad_len > MLXSW_EMAD_MAX_FRAME_LEN)
939 		return NULL;
940 
941 	skb = netdev_alloc_skb(NULL, emad_len);
942 	if (!skb)
943 		return NULL;
944 	memset(skb->data, 0, emad_len);
945 	skb_reserve(skb, emad_len);
946 
947 	return skb;
948 }
949 
950 static int mlxsw_emad_reg_access(struct mlxsw_core *mlxsw_core,
951 				 const struct mlxsw_reg_info *reg,
952 				 char *payload,
953 				 enum mlxsw_core_reg_access_type type,
954 				 struct mlxsw_reg_trans *trans,
955 				 struct list_head *bulk_list,
956 				 mlxsw_reg_trans_cb_t *cb,
957 				 unsigned long cb_priv, u64 tid)
958 {
959 	struct sk_buff *skb;
960 	int err;
961 
962 	dev_dbg(mlxsw_core->bus_info->dev, "EMAD reg access (tid=%llx,reg_id=%x(%s),type=%s)\n",
963 		tid, reg->id, mlxsw_reg_id_str(reg->id),
964 		mlxsw_core_reg_access_type_str(type));
965 
966 	skb = mlxsw_emad_alloc(mlxsw_core, reg->len);
967 	if (!skb)
968 		return -ENOMEM;
969 
970 	list_add_tail(&trans->bulk_list, bulk_list);
971 	trans->core = mlxsw_core;
972 	trans->tx_skb = skb;
973 	trans->tx_info.local_port = MLXSW_PORT_CPU_PORT;
974 	trans->tx_info.is_emad = true;
975 	INIT_DELAYED_WORK(&trans->timeout_dw, mlxsw_emad_trans_timeout_work);
976 	trans->tid = tid;
977 	init_completion(&trans->completion);
978 	trans->cb = cb;
979 	trans->cb_priv = cb_priv;
980 	trans->reg = reg;
981 	trans->type = type;
982 
983 	mlxsw_emad_construct(mlxsw_core, skb, reg, payload, type, trans->tid);
984 	mlxsw_core->driver->txhdr_construct(skb, &trans->tx_info);
985 
986 	spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
987 	list_add_tail_rcu(&trans->list, &mlxsw_core->emad.trans_list);
988 	spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
989 	err = mlxsw_emad_transmit(mlxsw_core, trans);
990 	if (err)
991 		goto err_out;
992 	return 0;
993 
994 err_out:
995 	spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
996 	list_del_rcu(&trans->list);
997 	spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
998 	list_del(&trans->bulk_list);
999 	dev_kfree_skb(trans->tx_skb);
1000 	return err;
1001 }
1002 
1003 /*****************
1004  * Core functions
1005  *****************/
1006 
1007 int mlxsw_core_driver_register(struct mlxsw_driver *mlxsw_driver)
1008 {
1009 	spin_lock(&mlxsw_core_driver_list_lock);
1010 	list_add_tail(&mlxsw_driver->list, &mlxsw_core_driver_list);
1011 	spin_unlock(&mlxsw_core_driver_list_lock);
1012 	return 0;
1013 }
1014 EXPORT_SYMBOL(mlxsw_core_driver_register);
1015 
1016 void mlxsw_core_driver_unregister(struct mlxsw_driver *mlxsw_driver)
1017 {
1018 	spin_lock(&mlxsw_core_driver_list_lock);
1019 	list_del(&mlxsw_driver->list);
1020 	spin_unlock(&mlxsw_core_driver_list_lock);
1021 }
1022 EXPORT_SYMBOL(mlxsw_core_driver_unregister);
1023 
1024 static struct mlxsw_driver *__driver_find(const char *kind)
1025 {
1026 	struct mlxsw_driver *mlxsw_driver;
1027 
1028 	list_for_each_entry(mlxsw_driver, &mlxsw_core_driver_list, list) {
1029 		if (strcmp(mlxsw_driver->kind, kind) == 0)
1030 			return mlxsw_driver;
1031 	}
1032 	return NULL;
1033 }
1034 
1035 static struct mlxsw_driver *mlxsw_core_driver_get(const char *kind)
1036 {
1037 	struct mlxsw_driver *mlxsw_driver;
1038 
1039 	spin_lock(&mlxsw_core_driver_list_lock);
1040 	mlxsw_driver = __driver_find(kind);
1041 	spin_unlock(&mlxsw_core_driver_list_lock);
1042 	return mlxsw_driver;
1043 }
1044 
1045 int mlxsw_core_fw_flash(struct mlxsw_core *mlxsw_core,
1046 			struct mlxfw_dev *mlxfw_dev,
1047 			const struct firmware *firmware,
1048 			struct netlink_ext_ack *extack)
1049 {
1050 	int err;
1051 
1052 	mlxsw_core->fw_flash_in_progress = true;
1053 	err = mlxfw_firmware_flash(mlxfw_dev, firmware, extack);
1054 	mlxsw_core->fw_flash_in_progress = false;
1055 
1056 	return err;
1057 }
1058 
1059 struct mlxsw_core_fw_info {
1060 	struct mlxfw_dev mlxfw_dev;
1061 	struct mlxsw_core *mlxsw_core;
1062 };
1063 
1064 static int mlxsw_core_fw_component_query(struct mlxfw_dev *mlxfw_dev,
1065 					 u16 component_index, u32 *p_max_size,
1066 					 u8 *p_align_bits, u16 *p_max_write_size)
1067 {
1068 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1069 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1070 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1071 	char mcqi_pl[MLXSW_REG_MCQI_LEN];
1072 	int err;
1073 
1074 	mlxsw_reg_mcqi_pack(mcqi_pl, component_index);
1075 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mcqi), mcqi_pl);
1076 	if (err)
1077 		return err;
1078 	mlxsw_reg_mcqi_unpack(mcqi_pl, p_max_size, p_align_bits, p_max_write_size);
1079 
1080 	*p_align_bits = max_t(u8, *p_align_bits, 2);
1081 	*p_max_write_size = min_t(u16, *p_max_write_size, MLXSW_REG_MCDA_MAX_DATA_LEN);
1082 	return 0;
1083 }
1084 
1085 static int mlxsw_core_fw_fsm_lock(struct mlxfw_dev *mlxfw_dev, u32 *fwhandle)
1086 {
1087 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1088 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1089 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1090 	char mcc_pl[MLXSW_REG_MCC_LEN];
1091 	u8 control_state;
1092 	int err;
1093 
1094 	mlxsw_reg_mcc_pack(mcc_pl, 0, 0, 0, 0);
1095 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1096 	if (err)
1097 		return err;
1098 
1099 	mlxsw_reg_mcc_unpack(mcc_pl, fwhandle, NULL, &control_state);
1100 	if (control_state != MLXFW_FSM_STATE_IDLE)
1101 		return -EBUSY;
1102 
1103 	mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_LOCK_UPDATE_HANDLE, 0, *fwhandle, 0);
1104 	return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1105 }
1106 
1107 static int mlxsw_core_fw_fsm_component_update(struct mlxfw_dev *mlxfw_dev, u32 fwhandle,
1108 					      u16 component_index, u32 component_size)
1109 {
1110 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1111 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1112 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1113 	char mcc_pl[MLXSW_REG_MCC_LEN];
1114 
1115 	mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_UPDATE_COMPONENT,
1116 			   component_index, fwhandle, component_size);
1117 	return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1118 }
1119 
1120 static int mlxsw_core_fw_fsm_block_download(struct mlxfw_dev *mlxfw_dev, u32 fwhandle,
1121 					    u8 *data, u16 size, u32 offset)
1122 {
1123 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1124 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1125 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1126 	char mcda_pl[MLXSW_REG_MCDA_LEN];
1127 
1128 	mlxsw_reg_mcda_pack(mcda_pl, fwhandle, offset, size, data);
1129 	return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcda), mcda_pl);
1130 }
1131 
1132 static int mlxsw_core_fw_fsm_component_verify(struct mlxfw_dev *mlxfw_dev, u32 fwhandle,
1133 					      u16 component_index)
1134 {
1135 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1136 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1137 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1138 	char mcc_pl[MLXSW_REG_MCC_LEN];
1139 
1140 	mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_VERIFY_COMPONENT,
1141 			   component_index, fwhandle, 0);
1142 	return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1143 }
1144 
1145 static int mlxsw_core_fw_fsm_activate(struct mlxfw_dev *mlxfw_dev, u32 fwhandle)
1146 {
1147 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1148 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1149 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1150 	char mcc_pl[MLXSW_REG_MCC_LEN];
1151 
1152 	mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_ACTIVATE, 0, fwhandle, 0);
1153 	return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1154 }
1155 
1156 static int mlxsw_core_fw_fsm_query_state(struct mlxfw_dev *mlxfw_dev, u32 fwhandle,
1157 					 enum mlxfw_fsm_state *fsm_state,
1158 					 enum mlxfw_fsm_state_err *fsm_state_err)
1159 {
1160 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1161 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1162 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1163 	char mcc_pl[MLXSW_REG_MCC_LEN];
1164 	u8 control_state;
1165 	u8 error_code;
1166 	int err;
1167 
1168 	mlxsw_reg_mcc_pack(mcc_pl, 0, 0, fwhandle, 0);
1169 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1170 	if (err)
1171 		return err;
1172 
1173 	mlxsw_reg_mcc_unpack(mcc_pl, NULL, &error_code, &control_state);
1174 	*fsm_state = control_state;
1175 	*fsm_state_err = min_t(enum mlxfw_fsm_state_err, error_code, MLXFW_FSM_STATE_ERR_MAX);
1176 	return 0;
1177 }
1178 
1179 static void mlxsw_core_fw_fsm_cancel(struct mlxfw_dev *mlxfw_dev, u32 fwhandle)
1180 {
1181 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1182 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1183 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1184 	char mcc_pl[MLXSW_REG_MCC_LEN];
1185 
1186 	mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_CANCEL, 0, fwhandle, 0);
1187 	mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1188 }
1189 
1190 static void mlxsw_core_fw_fsm_release(struct mlxfw_dev *mlxfw_dev, u32 fwhandle)
1191 {
1192 	struct mlxsw_core_fw_info *mlxsw_core_fw_info =
1193 		container_of(mlxfw_dev, struct mlxsw_core_fw_info, mlxfw_dev);
1194 	struct mlxsw_core *mlxsw_core = mlxsw_core_fw_info->mlxsw_core;
1195 	char mcc_pl[MLXSW_REG_MCC_LEN];
1196 
1197 	mlxsw_reg_mcc_pack(mcc_pl, MLXSW_REG_MCC_INSTRUCTION_RELEASE_UPDATE_HANDLE, 0, fwhandle, 0);
1198 	mlxsw_reg_write(mlxsw_core, MLXSW_REG(mcc), mcc_pl);
1199 }
1200 
1201 static const struct mlxfw_dev_ops mlxsw_core_fw_mlxsw_dev_ops = {
1202 	.component_query	= mlxsw_core_fw_component_query,
1203 	.fsm_lock		= mlxsw_core_fw_fsm_lock,
1204 	.fsm_component_update	= mlxsw_core_fw_fsm_component_update,
1205 	.fsm_block_download	= mlxsw_core_fw_fsm_block_download,
1206 	.fsm_component_verify	= mlxsw_core_fw_fsm_component_verify,
1207 	.fsm_activate		= mlxsw_core_fw_fsm_activate,
1208 	.fsm_query_state	= mlxsw_core_fw_fsm_query_state,
1209 	.fsm_cancel		= mlxsw_core_fw_fsm_cancel,
1210 	.fsm_release		= mlxsw_core_fw_fsm_release,
1211 };
1212 
1213 static int mlxsw_core_dev_fw_flash(struct mlxsw_core *mlxsw_core,
1214 				   const struct firmware *firmware,
1215 				   struct netlink_ext_ack *extack)
1216 {
1217 	struct mlxsw_core_fw_info mlxsw_core_fw_info = {
1218 		.mlxfw_dev = {
1219 			.ops = &mlxsw_core_fw_mlxsw_dev_ops,
1220 			.psid = mlxsw_core->bus_info->psid,
1221 			.psid_size = strlen(mlxsw_core->bus_info->psid),
1222 			.devlink = priv_to_devlink(mlxsw_core),
1223 		},
1224 		.mlxsw_core = mlxsw_core
1225 	};
1226 
1227 	return mlxsw_core_fw_flash(mlxsw_core, &mlxsw_core_fw_info.mlxfw_dev,
1228 				   firmware, extack);
1229 }
1230 
1231 static int mlxsw_core_fw_rev_validate(struct mlxsw_core *mlxsw_core,
1232 				      const struct mlxsw_bus_info *mlxsw_bus_info,
1233 				      const struct mlxsw_fw_rev *req_rev,
1234 				      const char *filename)
1235 {
1236 	const struct mlxsw_fw_rev *rev = &mlxsw_bus_info->fw_rev;
1237 	union devlink_param_value value;
1238 	const struct firmware *firmware;
1239 	int err;
1240 
1241 	/* Don't check if driver does not require it */
1242 	if (!req_rev || !filename)
1243 		return 0;
1244 
1245 	/* Don't check if devlink 'fw_load_policy' param is 'flash' */
1246 	err = devl_param_driverinit_value_get(priv_to_devlink(mlxsw_core),
1247 					      DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
1248 					      &value);
1249 	if (err)
1250 		return err;
1251 	if (value.vu8 == DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_FLASH)
1252 		return 0;
1253 
1254 	/* Validate driver & FW are compatible */
1255 	if (rev->major != req_rev->major) {
1256 		WARN(1, "Mismatch in major FW version [%d:%d] is never expected; Please contact support\n",
1257 		     rev->major, req_rev->major);
1258 		return -EINVAL;
1259 	}
1260 	if (mlxsw_core_fw_rev_minor_subminor_validate(rev, req_rev))
1261 		return 0;
1262 
1263 	dev_err(mlxsw_bus_info->dev, "The firmware version %d.%d.%d is incompatible with the driver (required >= %d.%d.%d)\n",
1264 		rev->major, rev->minor, rev->subminor, req_rev->major,
1265 		req_rev->minor, req_rev->subminor);
1266 	dev_info(mlxsw_bus_info->dev, "Flashing firmware using file %s\n", filename);
1267 
1268 	err = request_firmware_direct(&firmware, filename, mlxsw_bus_info->dev);
1269 	if (err) {
1270 		dev_err(mlxsw_bus_info->dev, "Could not request firmware file %s\n", filename);
1271 		return err;
1272 	}
1273 
1274 	err = mlxsw_core_dev_fw_flash(mlxsw_core, firmware, NULL);
1275 	release_firmware(firmware);
1276 	if (err)
1277 		dev_err(mlxsw_bus_info->dev, "Could not upgrade firmware\n");
1278 
1279 	/* On FW flash success, tell the caller FW reset is needed
1280 	 * if current FW supports it.
1281 	 */
1282 	if (rev->minor >= req_rev->can_reset_minor)
1283 		return err ? err : -EAGAIN;
1284 	else
1285 		return 0;
1286 }
1287 
1288 static int mlxsw_core_fw_flash_update(struct mlxsw_core *mlxsw_core,
1289 				      struct devlink_flash_update_params *params,
1290 				      struct netlink_ext_ack *extack)
1291 {
1292 	return mlxsw_core_dev_fw_flash(mlxsw_core, params->fw, extack);
1293 }
1294 
1295 static int mlxsw_core_devlink_param_fw_load_policy_validate(struct devlink *devlink, u32 id,
1296 							    union devlink_param_value val,
1297 							    struct netlink_ext_ack *extack)
1298 {
1299 	if (val.vu8 != DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_DRIVER &&
1300 	    val.vu8 != DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_FLASH) {
1301 		NL_SET_ERR_MSG_MOD(extack, "'fw_load_policy' must be 'driver' or 'flash'");
1302 		return -EINVAL;
1303 	}
1304 
1305 	return 0;
1306 }
1307 
1308 static const struct devlink_param mlxsw_core_fw_devlink_params[] = {
1309 	DEVLINK_PARAM_GENERIC(FW_LOAD_POLICY, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL,
1310 			      mlxsw_core_devlink_param_fw_load_policy_validate),
1311 };
1312 
1313 static int mlxsw_core_fw_params_register(struct mlxsw_core *mlxsw_core)
1314 {
1315 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
1316 	union devlink_param_value value;
1317 	int err;
1318 
1319 	err = devl_params_register(devlink, mlxsw_core_fw_devlink_params,
1320 				   ARRAY_SIZE(mlxsw_core_fw_devlink_params));
1321 	if (err)
1322 		return err;
1323 
1324 	value.vu8 = DEVLINK_PARAM_FW_LOAD_POLICY_VALUE_DRIVER;
1325 	devl_param_driverinit_value_set(devlink,
1326 					DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
1327 					value);
1328 	return 0;
1329 }
1330 
1331 static void mlxsw_core_fw_params_unregister(struct mlxsw_core *mlxsw_core)
1332 {
1333 	devl_params_unregister(priv_to_devlink(mlxsw_core), mlxsw_core_fw_devlink_params,
1334 			       ARRAY_SIZE(mlxsw_core_fw_devlink_params));
1335 }
1336 
1337 static void *__dl_port(struct devlink_port *devlink_port)
1338 {
1339 	return container_of(devlink_port, struct mlxsw_core_port, devlink_port);
1340 }
1341 
1342 static int mlxsw_devlink_port_split(struct devlink *devlink,
1343 				    struct devlink_port *port,
1344 				    unsigned int count,
1345 				    struct netlink_ext_ack *extack)
1346 {
1347 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(port);
1348 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1349 
1350 	if (!mlxsw_core->driver->port_split)
1351 		return -EOPNOTSUPP;
1352 	return mlxsw_core->driver->port_split(mlxsw_core,
1353 					      mlxsw_core_port->local_port,
1354 					      count, extack);
1355 }
1356 
1357 static int mlxsw_devlink_port_unsplit(struct devlink *devlink,
1358 				      struct devlink_port *port,
1359 				      struct netlink_ext_ack *extack)
1360 {
1361 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(port);
1362 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1363 
1364 	if (!mlxsw_core->driver->port_unsplit)
1365 		return -EOPNOTSUPP;
1366 	return mlxsw_core->driver->port_unsplit(mlxsw_core,
1367 						mlxsw_core_port->local_port,
1368 						extack);
1369 }
1370 
1371 static int
1372 mlxsw_devlink_sb_pool_get(struct devlink *devlink,
1373 			  unsigned int sb_index, u16 pool_index,
1374 			  struct devlink_sb_pool_info *pool_info)
1375 {
1376 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1377 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1378 
1379 	if (!mlxsw_driver->sb_pool_get)
1380 		return -EOPNOTSUPP;
1381 	return mlxsw_driver->sb_pool_get(mlxsw_core, sb_index,
1382 					 pool_index, pool_info);
1383 }
1384 
1385 static int
1386 mlxsw_devlink_sb_pool_set(struct devlink *devlink,
1387 			  unsigned int sb_index, u16 pool_index, u32 size,
1388 			  enum devlink_sb_threshold_type threshold_type,
1389 			  struct netlink_ext_ack *extack)
1390 {
1391 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1392 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1393 
1394 	if (!mlxsw_driver->sb_pool_set)
1395 		return -EOPNOTSUPP;
1396 	return mlxsw_driver->sb_pool_set(mlxsw_core, sb_index,
1397 					 pool_index, size, threshold_type,
1398 					 extack);
1399 }
1400 
1401 static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port,
1402 					  unsigned int sb_index, u16 pool_index,
1403 					  u32 *p_threshold)
1404 {
1405 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1406 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1407 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1408 
1409 	if (!mlxsw_driver->sb_port_pool_get ||
1410 	    !mlxsw_core_port_check(mlxsw_core_port))
1411 		return -EOPNOTSUPP;
1412 	return mlxsw_driver->sb_port_pool_get(mlxsw_core_port, sb_index,
1413 					      pool_index, p_threshold);
1414 }
1415 
1416 static int mlxsw_devlink_sb_port_pool_set(struct devlink_port *devlink_port,
1417 					  unsigned int sb_index, u16 pool_index,
1418 					  u32 threshold,
1419 					  struct netlink_ext_ack *extack)
1420 {
1421 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1422 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1423 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1424 
1425 	if (!mlxsw_driver->sb_port_pool_set ||
1426 	    !mlxsw_core_port_check(mlxsw_core_port))
1427 		return -EOPNOTSUPP;
1428 	return mlxsw_driver->sb_port_pool_set(mlxsw_core_port, sb_index,
1429 					      pool_index, threshold, extack);
1430 }
1431 
1432 static int
1433 mlxsw_devlink_sb_tc_pool_bind_get(struct devlink_port *devlink_port,
1434 				  unsigned int sb_index, u16 tc_index,
1435 				  enum devlink_sb_pool_type pool_type,
1436 				  u16 *p_pool_index, u32 *p_threshold)
1437 {
1438 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1439 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1440 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1441 
1442 	if (!mlxsw_driver->sb_tc_pool_bind_get ||
1443 	    !mlxsw_core_port_check(mlxsw_core_port))
1444 		return -EOPNOTSUPP;
1445 	return mlxsw_driver->sb_tc_pool_bind_get(mlxsw_core_port, sb_index,
1446 						 tc_index, pool_type,
1447 						 p_pool_index, p_threshold);
1448 }
1449 
1450 static int
1451 mlxsw_devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
1452 				  unsigned int sb_index, u16 tc_index,
1453 				  enum devlink_sb_pool_type pool_type,
1454 				  u16 pool_index, u32 threshold,
1455 				  struct netlink_ext_ack *extack)
1456 {
1457 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1458 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1459 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1460 
1461 	if (!mlxsw_driver->sb_tc_pool_bind_set ||
1462 	    !mlxsw_core_port_check(mlxsw_core_port))
1463 		return -EOPNOTSUPP;
1464 	return mlxsw_driver->sb_tc_pool_bind_set(mlxsw_core_port, sb_index,
1465 						 tc_index, pool_type,
1466 						 pool_index, threshold, extack);
1467 }
1468 
1469 static int mlxsw_devlink_sb_occ_snapshot(struct devlink *devlink,
1470 					 unsigned int sb_index)
1471 {
1472 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1473 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1474 
1475 	if (!mlxsw_driver->sb_occ_snapshot)
1476 		return -EOPNOTSUPP;
1477 	return mlxsw_driver->sb_occ_snapshot(mlxsw_core, sb_index);
1478 }
1479 
1480 static int mlxsw_devlink_sb_occ_max_clear(struct devlink *devlink,
1481 					  unsigned int sb_index)
1482 {
1483 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1484 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1485 
1486 	if (!mlxsw_driver->sb_occ_max_clear)
1487 		return -EOPNOTSUPP;
1488 	return mlxsw_driver->sb_occ_max_clear(mlxsw_core, sb_index);
1489 }
1490 
1491 static int
1492 mlxsw_devlink_sb_occ_port_pool_get(struct devlink_port *devlink_port,
1493 				   unsigned int sb_index, u16 pool_index,
1494 				   u32 *p_cur, u32 *p_max)
1495 {
1496 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1497 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1498 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1499 
1500 	if (!mlxsw_driver->sb_occ_port_pool_get ||
1501 	    !mlxsw_core_port_check(mlxsw_core_port))
1502 		return -EOPNOTSUPP;
1503 	return mlxsw_driver->sb_occ_port_pool_get(mlxsw_core_port, sb_index,
1504 						  pool_index, p_cur, p_max);
1505 }
1506 
1507 static int
1508 mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port,
1509 				      unsigned int sb_index, u16 tc_index,
1510 				      enum devlink_sb_pool_type pool_type,
1511 				      u32 *p_cur, u32 *p_max)
1512 {
1513 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1514 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1515 	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1516 
1517 	if (!mlxsw_driver->sb_occ_tc_port_bind_get ||
1518 	    !mlxsw_core_port_check(mlxsw_core_port))
1519 		return -EOPNOTSUPP;
1520 	return mlxsw_driver->sb_occ_tc_port_bind_get(mlxsw_core_port,
1521 						     sb_index, tc_index,
1522 						     pool_type, p_cur, p_max);
1523 }
1524 
1525 static int
1526 mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req,
1527 		       struct netlink_ext_ack *extack)
1528 {
1529 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1530 	char fw_info_psid[MLXSW_REG_MGIR_FW_INFO_PSID_SIZE];
1531 	u32 hw_rev, fw_major, fw_minor, fw_sub_minor;
1532 	char mgir_pl[MLXSW_REG_MGIR_LEN];
1533 	char buf[32];
1534 	int err;
1535 
1536 	mlxsw_reg_mgir_pack(mgir_pl);
1537 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mgir), mgir_pl);
1538 	if (err)
1539 		return err;
1540 	mlxsw_reg_mgir_unpack(mgir_pl, &hw_rev, fw_info_psid, &fw_major,
1541 			      &fw_minor, &fw_sub_minor);
1542 
1543 	sprintf(buf, "%X", hw_rev);
1544 	err = devlink_info_version_fixed_put(req, "hw.revision", buf);
1545 	if (err)
1546 		return err;
1547 
1548 	err = devlink_info_version_fixed_put(req,
1549 					     DEVLINK_INFO_VERSION_GENERIC_FW_PSID,
1550 					     fw_info_psid);
1551 	if (err)
1552 		return err;
1553 
1554 	sprintf(buf, "%d.%d.%d", fw_major, fw_minor, fw_sub_minor);
1555 	err = devlink_info_version_running_put(req, "fw.version", buf);
1556 	if (err)
1557 		return err;
1558 
1559 	return devlink_info_version_running_put(req,
1560 						DEVLINK_INFO_VERSION_GENERIC_FW,
1561 						buf);
1562 }
1563 
1564 static int
1565 mlxsw_devlink_core_bus_device_reload_down(struct devlink *devlink,
1566 					  bool netns_change, enum devlink_reload_action action,
1567 					  enum devlink_reload_limit limit,
1568 					  struct netlink_ext_ack *extack)
1569 {
1570 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1571 
1572 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_RESET))
1573 		return -EOPNOTSUPP;
1574 
1575 	mlxsw_core_bus_device_unregister(mlxsw_core, true);
1576 	return 0;
1577 }
1578 
1579 static int
1580 mlxsw_devlink_core_bus_device_reload_up(struct devlink *devlink, enum devlink_reload_action action,
1581 					enum devlink_reload_limit limit, u32 *actions_performed,
1582 					struct netlink_ext_ack *extack)
1583 {
1584 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1585 	int err;
1586 
1587 	*actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
1588 			     BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE);
1589 	err = mlxsw_core_bus_device_register(mlxsw_core->bus_info,
1590 					     mlxsw_core->bus,
1591 					     mlxsw_core->bus_priv, true,
1592 					     devlink, extack);
1593 	return err;
1594 }
1595 
1596 static int mlxsw_devlink_flash_update(struct devlink *devlink,
1597 				      struct devlink_flash_update_params *params,
1598 				      struct netlink_ext_ack *extack)
1599 {
1600 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1601 
1602 	return mlxsw_core_fw_flash_update(mlxsw_core, params, extack);
1603 }
1604 
1605 static int mlxsw_devlink_trap_init(struct devlink *devlink,
1606 				   const struct devlink_trap *trap,
1607 				   void *trap_ctx)
1608 {
1609 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1610 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1611 
1612 	if (!mlxsw_driver->trap_init)
1613 		return -EOPNOTSUPP;
1614 	return mlxsw_driver->trap_init(mlxsw_core, trap, trap_ctx);
1615 }
1616 
1617 static void mlxsw_devlink_trap_fini(struct devlink *devlink,
1618 				    const struct devlink_trap *trap,
1619 				    void *trap_ctx)
1620 {
1621 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1622 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1623 
1624 	if (!mlxsw_driver->trap_fini)
1625 		return;
1626 	mlxsw_driver->trap_fini(mlxsw_core, trap, trap_ctx);
1627 }
1628 
1629 static int mlxsw_devlink_trap_action_set(struct devlink *devlink,
1630 					 const struct devlink_trap *trap,
1631 					 enum devlink_trap_action action,
1632 					 struct netlink_ext_ack *extack)
1633 {
1634 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1635 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1636 
1637 	if (!mlxsw_driver->trap_action_set)
1638 		return -EOPNOTSUPP;
1639 	return mlxsw_driver->trap_action_set(mlxsw_core, trap, action, extack);
1640 }
1641 
1642 static int
1643 mlxsw_devlink_trap_group_init(struct devlink *devlink,
1644 			      const struct devlink_trap_group *group)
1645 {
1646 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1647 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1648 
1649 	if (!mlxsw_driver->trap_group_init)
1650 		return -EOPNOTSUPP;
1651 	return mlxsw_driver->trap_group_init(mlxsw_core, group);
1652 }
1653 
1654 static int
1655 mlxsw_devlink_trap_group_set(struct devlink *devlink,
1656 			     const struct devlink_trap_group *group,
1657 			     const struct devlink_trap_policer *policer,
1658 			     struct netlink_ext_ack *extack)
1659 {
1660 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1661 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1662 
1663 	if (!mlxsw_driver->trap_group_set)
1664 		return -EOPNOTSUPP;
1665 	return mlxsw_driver->trap_group_set(mlxsw_core, group, policer, extack);
1666 }
1667 
1668 static int
1669 mlxsw_devlink_trap_policer_init(struct devlink *devlink,
1670 				const struct devlink_trap_policer *policer)
1671 {
1672 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1673 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1674 
1675 	if (!mlxsw_driver->trap_policer_init)
1676 		return -EOPNOTSUPP;
1677 	return mlxsw_driver->trap_policer_init(mlxsw_core, policer);
1678 }
1679 
1680 static void
1681 mlxsw_devlink_trap_policer_fini(struct devlink *devlink,
1682 				const struct devlink_trap_policer *policer)
1683 {
1684 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1685 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1686 
1687 	if (!mlxsw_driver->trap_policer_fini)
1688 		return;
1689 	mlxsw_driver->trap_policer_fini(mlxsw_core, policer);
1690 }
1691 
1692 static int
1693 mlxsw_devlink_trap_policer_set(struct devlink *devlink,
1694 			       const struct devlink_trap_policer *policer,
1695 			       u64 rate, u64 burst,
1696 			       struct netlink_ext_ack *extack)
1697 {
1698 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1699 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1700 
1701 	if (!mlxsw_driver->trap_policer_set)
1702 		return -EOPNOTSUPP;
1703 	return mlxsw_driver->trap_policer_set(mlxsw_core, policer, rate, burst,
1704 					      extack);
1705 }
1706 
1707 static int
1708 mlxsw_devlink_trap_policer_counter_get(struct devlink *devlink,
1709 				       const struct devlink_trap_policer *policer,
1710 				       u64 *p_drops)
1711 {
1712 	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1713 	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1714 
1715 	if (!mlxsw_driver->trap_policer_counter_get)
1716 		return -EOPNOTSUPP;
1717 	return mlxsw_driver->trap_policer_counter_get(mlxsw_core, policer,
1718 						      p_drops);
1719 }
1720 
1721 static const struct devlink_ops mlxsw_devlink_ops = {
1722 	.reload_actions		= BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
1723 				  BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE),
1724 	.reload_down		= mlxsw_devlink_core_bus_device_reload_down,
1725 	.reload_up		= mlxsw_devlink_core_bus_device_reload_up,
1726 	.sb_pool_get			= mlxsw_devlink_sb_pool_get,
1727 	.sb_pool_set			= mlxsw_devlink_sb_pool_set,
1728 	.sb_port_pool_get		= mlxsw_devlink_sb_port_pool_get,
1729 	.sb_port_pool_set		= mlxsw_devlink_sb_port_pool_set,
1730 	.sb_tc_pool_bind_get		= mlxsw_devlink_sb_tc_pool_bind_get,
1731 	.sb_tc_pool_bind_set		= mlxsw_devlink_sb_tc_pool_bind_set,
1732 	.sb_occ_snapshot		= mlxsw_devlink_sb_occ_snapshot,
1733 	.sb_occ_max_clear		= mlxsw_devlink_sb_occ_max_clear,
1734 	.sb_occ_port_pool_get		= mlxsw_devlink_sb_occ_port_pool_get,
1735 	.sb_occ_tc_port_bind_get	= mlxsw_devlink_sb_occ_tc_port_bind_get,
1736 	.info_get			= mlxsw_devlink_info_get,
1737 	.flash_update			= mlxsw_devlink_flash_update,
1738 	.trap_init			= mlxsw_devlink_trap_init,
1739 	.trap_fini			= mlxsw_devlink_trap_fini,
1740 	.trap_action_set		= mlxsw_devlink_trap_action_set,
1741 	.trap_group_init		= mlxsw_devlink_trap_group_init,
1742 	.trap_group_set			= mlxsw_devlink_trap_group_set,
1743 	.trap_policer_init		= mlxsw_devlink_trap_policer_init,
1744 	.trap_policer_fini		= mlxsw_devlink_trap_policer_fini,
1745 	.trap_policer_set		= mlxsw_devlink_trap_policer_set,
1746 	.trap_policer_counter_get	= mlxsw_devlink_trap_policer_counter_get,
1747 };
1748 
1749 static int mlxsw_core_params_register(struct mlxsw_core *mlxsw_core)
1750 {
1751 	return mlxsw_core_fw_params_register(mlxsw_core);
1752 }
1753 
1754 static void mlxsw_core_params_unregister(struct mlxsw_core *mlxsw_core)
1755 {
1756 	mlxsw_core_fw_params_unregister(mlxsw_core);
1757 }
1758 
1759 struct mlxsw_core_health_event {
1760 	struct mlxsw_core *mlxsw_core;
1761 	char mfde_pl[MLXSW_REG_MFDE_LEN];
1762 	struct work_struct work;
1763 };
1764 
1765 static void mlxsw_core_health_event_work(struct work_struct *work)
1766 {
1767 	struct mlxsw_core_health_event *event;
1768 	struct mlxsw_core *mlxsw_core;
1769 
1770 	event = container_of(work, struct mlxsw_core_health_event, work);
1771 	mlxsw_core = event->mlxsw_core;
1772 	devlink_health_report(mlxsw_core->health.fw_fatal, "FW fatal event occurred",
1773 			      event->mfde_pl);
1774 	kfree(event);
1775 }
1776 
1777 static void mlxsw_core_health_listener_func(const struct mlxsw_reg_info *reg,
1778 					    char *mfde_pl, void *priv)
1779 {
1780 	struct mlxsw_core_health_event *event;
1781 	struct mlxsw_core *mlxsw_core = priv;
1782 
1783 	event = kmalloc(sizeof(*event), GFP_ATOMIC);
1784 	if (!event)
1785 		return;
1786 	event->mlxsw_core = mlxsw_core;
1787 	memcpy(event->mfde_pl, mfde_pl, sizeof(event->mfde_pl));
1788 	INIT_WORK(&event->work, mlxsw_core_health_event_work);
1789 	mlxsw_core_schedule_work(&event->work);
1790 }
1791 
1792 static const struct mlxsw_listener mlxsw_core_health_listener =
1793 	MLXSW_CORE_EVENTL(mlxsw_core_health_listener_func, MFDE);
1794 
1795 static int
1796 mlxsw_core_health_fw_fatal_dump_fatal_cause(const char *mfde_pl,
1797 					    struct devlink_fmsg *fmsg)
1798 {
1799 	u32 val, tile_v;
1800 	int err;
1801 
1802 	val = mlxsw_reg_mfde_fatal_cause_id_get(mfde_pl);
1803 	err = devlink_fmsg_u32_pair_put(fmsg, "cause_id", val);
1804 	if (err)
1805 		return err;
1806 	tile_v = mlxsw_reg_mfde_fatal_cause_tile_v_get(mfde_pl);
1807 	if (tile_v) {
1808 		val = mlxsw_reg_mfde_fatal_cause_tile_index_get(mfde_pl);
1809 		err = devlink_fmsg_u8_pair_put(fmsg, "tile_index", val);
1810 		if (err)
1811 			return err;
1812 	}
1813 
1814 	return 0;
1815 }
1816 
1817 static int
1818 mlxsw_core_health_fw_fatal_dump_fw_assert(const char *mfde_pl,
1819 					  struct devlink_fmsg *fmsg)
1820 {
1821 	u32 val, tile_v;
1822 	int err;
1823 
1824 	val = mlxsw_reg_mfde_fw_assert_var0_get(mfde_pl);
1825 	err = devlink_fmsg_u32_pair_put(fmsg, "var0", val);
1826 	if (err)
1827 		return err;
1828 	val = mlxsw_reg_mfde_fw_assert_var1_get(mfde_pl);
1829 	err = devlink_fmsg_u32_pair_put(fmsg, "var1", val);
1830 	if (err)
1831 		return err;
1832 	val = mlxsw_reg_mfde_fw_assert_var2_get(mfde_pl);
1833 	err = devlink_fmsg_u32_pair_put(fmsg, "var2", val);
1834 	if (err)
1835 		return err;
1836 	val = mlxsw_reg_mfde_fw_assert_var3_get(mfde_pl);
1837 	err = devlink_fmsg_u32_pair_put(fmsg, "var3", val);
1838 	if (err)
1839 		return err;
1840 	val = mlxsw_reg_mfde_fw_assert_var4_get(mfde_pl);
1841 	err = devlink_fmsg_u32_pair_put(fmsg, "var4", val);
1842 	if (err)
1843 		return err;
1844 	val = mlxsw_reg_mfde_fw_assert_existptr_get(mfde_pl);
1845 	err = devlink_fmsg_u32_pair_put(fmsg, "existptr", val);
1846 	if (err)
1847 		return err;
1848 	val = mlxsw_reg_mfde_fw_assert_callra_get(mfde_pl);
1849 	err = devlink_fmsg_u32_pair_put(fmsg, "callra", val);
1850 	if (err)
1851 		return err;
1852 	val = mlxsw_reg_mfde_fw_assert_oe_get(mfde_pl);
1853 	err = devlink_fmsg_bool_pair_put(fmsg, "old_event", val);
1854 	if (err)
1855 		return err;
1856 	tile_v = mlxsw_reg_mfde_fw_assert_tile_v_get(mfde_pl);
1857 	if (tile_v) {
1858 		val = mlxsw_reg_mfde_fw_assert_tile_index_get(mfde_pl);
1859 		err = devlink_fmsg_u8_pair_put(fmsg, "tile_index", val);
1860 		if (err)
1861 			return err;
1862 	}
1863 	val = mlxsw_reg_mfde_fw_assert_ext_synd_get(mfde_pl);
1864 	err = devlink_fmsg_u32_pair_put(fmsg, "ext_synd", val);
1865 	if (err)
1866 		return err;
1867 
1868 	return 0;
1869 }
1870 
1871 static int
1872 mlxsw_core_health_fw_fatal_dump_kvd_im_stop(const char *mfde_pl,
1873 					    struct devlink_fmsg *fmsg)
1874 {
1875 	u32 val;
1876 	int err;
1877 
1878 	val = mlxsw_reg_mfde_kvd_im_stop_oe_get(mfde_pl);
1879 	err = devlink_fmsg_bool_pair_put(fmsg, "old_event", val);
1880 	if (err)
1881 		return err;
1882 	val = mlxsw_reg_mfde_kvd_im_stop_pipes_mask_get(mfde_pl);
1883 	return devlink_fmsg_u32_pair_put(fmsg, "pipes_mask", val);
1884 }
1885 
1886 static int
1887 mlxsw_core_health_fw_fatal_dump_crspace_to(const char *mfde_pl,
1888 					   struct devlink_fmsg *fmsg)
1889 {
1890 	u32 val;
1891 	int err;
1892 
1893 	val = mlxsw_reg_mfde_crspace_to_log_address_get(mfde_pl);
1894 	err = devlink_fmsg_u32_pair_put(fmsg, "log_address", val);
1895 	if (err)
1896 		return err;
1897 	val = mlxsw_reg_mfde_crspace_to_oe_get(mfde_pl);
1898 	err = devlink_fmsg_bool_pair_put(fmsg, "old_event", val);
1899 	if (err)
1900 		return err;
1901 	val = mlxsw_reg_mfde_crspace_to_log_id_get(mfde_pl);
1902 	err = devlink_fmsg_u8_pair_put(fmsg, "log_irisc_id", val);
1903 	if (err)
1904 		return err;
1905 	val = mlxsw_reg_mfde_crspace_to_log_ip_get(mfde_pl);
1906 	err = devlink_fmsg_u64_pair_put(fmsg, "log_ip", val);
1907 	if (err)
1908 		return err;
1909 
1910 	return 0;
1911 }
1912 
1913 static int mlxsw_core_health_fw_fatal_dump(struct devlink_health_reporter *reporter,
1914 					   struct devlink_fmsg *fmsg, void *priv_ctx,
1915 					   struct netlink_ext_ack *extack)
1916 {
1917 	char *mfde_pl = priv_ctx;
1918 	char *val_str;
1919 	u8 event_id;
1920 	u32 val;
1921 	int err;
1922 
1923 	if (!priv_ctx)
1924 		/* User-triggered dumps are not possible */
1925 		return -EOPNOTSUPP;
1926 
1927 	val = mlxsw_reg_mfde_irisc_id_get(mfde_pl);
1928 	err = devlink_fmsg_u8_pair_put(fmsg, "irisc_id", val);
1929 	if (err)
1930 		return err;
1931 	err = devlink_fmsg_arr_pair_nest_start(fmsg, "event");
1932 	if (err)
1933 		return err;
1934 
1935 	event_id = mlxsw_reg_mfde_event_id_get(mfde_pl);
1936 	err = devlink_fmsg_u32_pair_put(fmsg, "id", event_id);
1937 	if (err)
1938 		return err;
1939 	switch (event_id) {
1940 	case MLXSW_REG_MFDE_EVENT_ID_CRSPACE_TO:
1941 		val_str = "CR space timeout";
1942 		break;
1943 	case MLXSW_REG_MFDE_EVENT_ID_KVD_IM_STOP:
1944 		val_str = "KVD insertion machine stopped";
1945 		break;
1946 	case MLXSW_REG_MFDE_EVENT_ID_TEST:
1947 		val_str = "Test";
1948 		break;
1949 	case MLXSW_REG_MFDE_EVENT_ID_FW_ASSERT:
1950 		val_str = "FW assert";
1951 		break;
1952 	case MLXSW_REG_MFDE_EVENT_ID_FATAL_CAUSE:
1953 		val_str = "Fatal cause";
1954 		break;
1955 	default:
1956 		val_str = NULL;
1957 	}
1958 	if (val_str) {
1959 		err = devlink_fmsg_string_pair_put(fmsg, "desc", val_str);
1960 		if (err)
1961 			return err;
1962 	}
1963 
1964 	err = devlink_fmsg_arr_pair_nest_end(fmsg);
1965 	if (err)
1966 		return err;
1967 
1968 	err = devlink_fmsg_arr_pair_nest_start(fmsg, "severity");
1969 	if (err)
1970 		return err;
1971 
1972 	val = mlxsw_reg_mfde_severity_get(mfde_pl);
1973 	err = devlink_fmsg_u8_pair_put(fmsg, "id", val);
1974 	if (err)
1975 		return err;
1976 	switch (val) {
1977 	case MLXSW_REG_MFDE_SEVERITY_FATL:
1978 		val_str = "Fatal";
1979 		break;
1980 	case MLXSW_REG_MFDE_SEVERITY_NRML:
1981 		val_str = "Normal";
1982 		break;
1983 	case MLXSW_REG_MFDE_SEVERITY_INTR:
1984 		val_str = "Debug";
1985 		break;
1986 	default:
1987 		val_str = NULL;
1988 	}
1989 	if (val_str) {
1990 		err = devlink_fmsg_string_pair_put(fmsg, "desc", val_str);
1991 		if (err)
1992 			return err;
1993 	}
1994 
1995 	err = devlink_fmsg_arr_pair_nest_end(fmsg);
1996 	if (err)
1997 		return err;
1998 
1999 	val = mlxsw_reg_mfde_method_get(mfde_pl);
2000 	switch (val) {
2001 	case MLXSW_REG_MFDE_METHOD_QUERY:
2002 		val_str = "query";
2003 		break;
2004 	case MLXSW_REG_MFDE_METHOD_WRITE:
2005 		val_str = "write";
2006 		break;
2007 	default:
2008 		val_str = NULL;
2009 	}
2010 	if (val_str) {
2011 		err = devlink_fmsg_string_pair_put(fmsg, "method", val_str);
2012 		if (err)
2013 			return err;
2014 	}
2015 
2016 	val = mlxsw_reg_mfde_long_process_get(mfde_pl);
2017 	err = devlink_fmsg_bool_pair_put(fmsg, "long_process", val);
2018 	if (err)
2019 		return err;
2020 
2021 	val = mlxsw_reg_mfde_command_type_get(mfde_pl);
2022 	switch (val) {
2023 	case MLXSW_REG_MFDE_COMMAND_TYPE_MAD:
2024 		val_str = "mad";
2025 		break;
2026 	case MLXSW_REG_MFDE_COMMAND_TYPE_EMAD:
2027 		val_str = "emad";
2028 		break;
2029 	case MLXSW_REG_MFDE_COMMAND_TYPE_CMDIF:
2030 		val_str = "cmdif";
2031 		break;
2032 	default:
2033 		val_str = NULL;
2034 	}
2035 	if (val_str) {
2036 		err = devlink_fmsg_string_pair_put(fmsg, "command_type", val_str);
2037 		if (err)
2038 			return err;
2039 	}
2040 
2041 	val = mlxsw_reg_mfde_reg_attr_id_get(mfde_pl);
2042 	err = devlink_fmsg_u32_pair_put(fmsg, "reg_attr_id", val);
2043 	if (err)
2044 		return err;
2045 
2046 	switch (event_id) {
2047 	case MLXSW_REG_MFDE_EVENT_ID_CRSPACE_TO:
2048 		return mlxsw_core_health_fw_fatal_dump_crspace_to(mfde_pl,
2049 								  fmsg);
2050 	case MLXSW_REG_MFDE_EVENT_ID_KVD_IM_STOP:
2051 		return mlxsw_core_health_fw_fatal_dump_kvd_im_stop(mfde_pl,
2052 								   fmsg);
2053 	case MLXSW_REG_MFDE_EVENT_ID_FW_ASSERT:
2054 		return mlxsw_core_health_fw_fatal_dump_fw_assert(mfde_pl, fmsg);
2055 	case MLXSW_REG_MFDE_EVENT_ID_FATAL_CAUSE:
2056 		return mlxsw_core_health_fw_fatal_dump_fatal_cause(mfde_pl,
2057 								   fmsg);
2058 	}
2059 
2060 	return 0;
2061 }
2062 
2063 static int
2064 mlxsw_core_health_fw_fatal_test(struct devlink_health_reporter *reporter,
2065 				struct netlink_ext_ack *extack)
2066 {
2067 	struct mlxsw_core *mlxsw_core = devlink_health_reporter_priv(reporter);
2068 	char mfgd_pl[MLXSW_REG_MFGD_LEN];
2069 	int err;
2070 
2071 	/* Read the register first to make sure no other bits are changed. */
2072 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mfgd), mfgd_pl);
2073 	if (err)
2074 		return err;
2075 	mlxsw_reg_mfgd_trigger_test_set(mfgd_pl, true);
2076 	return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mfgd), mfgd_pl);
2077 }
2078 
2079 static const struct devlink_health_reporter_ops
2080 mlxsw_core_health_fw_fatal_ops = {
2081 	.name = "fw_fatal",
2082 	.dump = mlxsw_core_health_fw_fatal_dump,
2083 	.test = mlxsw_core_health_fw_fatal_test,
2084 };
2085 
2086 static int mlxsw_core_health_fw_fatal_config(struct mlxsw_core *mlxsw_core,
2087 					     bool enable)
2088 {
2089 	char mfgd_pl[MLXSW_REG_MFGD_LEN];
2090 	int err;
2091 
2092 	/* Read the register first to make sure no other bits are changed. */
2093 	err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mfgd), mfgd_pl);
2094 	if (err)
2095 		return err;
2096 	mlxsw_reg_mfgd_fatal_event_mode_set(mfgd_pl, enable);
2097 	return mlxsw_reg_write(mlxsw_core, MLXSW_REG(mfgd), mfgd_pl);
2098 }
2099 
2100 static int mlxsw_core_health_init(struct mlxsw_core *mlxsw_core)
2101 {
2102 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
2103 	struct devlink_health_reporter *fw_fatal;
2104 	int err;
2105 
2106 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
2107 		return 0;
2108 
2109 	fw_fatal = devl_health_reporter_create(devlink, &mlxsw_core_health_fw_fatal_ops,
2110 					       0, mlxsw_core);
2111 	if (IS_ERR(fw_fatal)) {
2112 		dev_err(mlxsw_core->bus_info->dev, "Failed to create fw fatal reporter");
2113 		return PTR_ERR(fw_fatal);
2114 	}
2115 	mlxsw_core->health.fw_fatal = fw_fatal;
2116 
2117 	err = mlxsw_core_trap_register(mlxsw_core, &mlxsw_core_health_listener, mlxsw_core);
2118 	if (err)
2119 		goto err_trap_register;
2120 
2121 	err = mlxsw_core_health_fw_fatal_config(mlxsw_core, true);
2122 	if (err)
2123 		goto err_fw_fatal_config;
2124 
2125 	return 0;
2126 
2127 err_fw_fatal_config:
2128 	mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_core_health_listener, mlxsw_core);
2129 err_trap_register:
2130 	devl_health_reporter_destroy(mlxsw_core->health.fw_fatal);
2131 	return err;
2132 }
2133 
2134 static void mlxsw_core_health_fini(struct mlxsw_core *mlxsw_core)
2135 {
2136 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
2137 		return;
2138 
2139 	mlxsw_core_health_fw_fatal_config(mlxsw_core, false);
2140 	mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_core_health_listener, mlxsw_core);
2141 	/* Make sure there is no more event work scheduled */
2142 	mlxsw_core_flush_owq();
2143 	devl_health_reporter_destroy(mlxsw_core->health.fw_fatal);
2144 }
2145 
2146 static void mlxsw_core_irq_event_handler_init(struct mlxsw_core *mlxsw_core)
2147 {
2148 	INIT_LIST_HEAD(&mlxsw_core->irq_event_handler_list);
2149 	mutex_init(&mlxsw_core->irq_event_handler_lock);
2150 }
2151 
2152 static void mlxsw_core_irq_event_handler_fini(struct mlxsw_core *mlxsw_core)
2153 {
2154 	mutex_destroy(&mlxsw_core->irq_event_handler_lock);
2155 	WARN_ON(!list_empty(&mlxsw_core->irq_event_handler_list));
2156 }
2157 
2158 static int
2159 __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
2160 				 const struct mlxsw_bus *mlxsw_bus,
2161 				 void *bus_priv, bool reload,
2162 				 struct devlink *devlink,
2163 				 struct netlink_ext_ack *extack)
2164 {
2165 	const char *device_kind = mlxsw_bus_info->device_kind;
2166 	struct mlxsw_core *mlxsw_core;
2167 	struct mlxsw_driver *mlxsw_driver;
2168 	size_t alloc_size;
2169 	u16 max_lag;
2170 	int err;
2171 
2172 	mlxsw_driver = mlxsw_core_driver_get(device_kind);
2173 	if (!mlxsw_driver)
2174 		return -EINVAL;
2175 
2176 	if (!reload) {
2177 		alloc_size = sizeof(*mlxsw_core) + mlxsw_driver->priv_size;
2178 		devlink = devlink_alloc(&mlxsw_devlink_ops, alloc_size,
2179 					mlxsw_bus_info->dev);
2180 		if (!devlink) {
2181 			err = -ENOMEM;
2182 			goto err_devlink_alloc;
2183 		}
2184 		devl_lock(devlink);
2185 		devl_register(devlink);
2186 	}
2187 
2188 	mlxsw_core = devlink_priv(devlink);
2189 	INIT_LIST_HEAD(&mlxsw_core->rx_listener_list);
2190 	INIT_LIST_HEAD(&mlxsw_core->event_listener_list);
2191 	mlxsw_core->driver = mlxsw_driver;
2192 	mlxsw_core->bus = mlxsw_bus;
2193 	mlxsw_core->bus_priv = bus_priv;
2194 	mlxsw_core->bus_info = mlxsw_bus_info;
2195 	mlxsw_core_irq_event_handler_init(mlxsw_core);
2196 
2197 	err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile,
2198 			      &mlxsw_core->res);
2199 	if (err)
2200 		goto err_bus_init;
2201 
2202 	if (mlxsw_driver->resources_register && !reload) {
2203 		err = mlxsw_driver->resources_register(mlxsw_core);
2204 		if (err)
2205 			goto err_register_resources;
2206 	}
2207 
2208 	err = mlxsw_ports_init(mlxsw_core, reload);
2209 	if (err)
2210 		goto err_ports_init;
2211 
2212 	err = mlxsw_core_max_lag(mlxsw_core, &max_lag);
2213 	if (!err && MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) {
2214 		alloc_size = sizeof(*mlxsw_core->lag.mapping) * max_lag *
2215 			MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS);
2216 		mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL);
2217 		if (!mlxsw_core->lag.mapping) {
2218 			err = -ENOMEM;
2219 			goto err_alloc_lag_mapping;
2220 		}
2221 	}
2222 
2223 	err = mlxsw_core_trap_groups_set(mlxsw_core);
2224 	if (err)
2225 		goto err_trap_groups_set;
2226 
2227 	err = mlxsw_emad_init(mlxsw_core);
2228 	if (err)
2229 		goto err_emad_init;
2230 
2231 	if (!reload) {
2232 		err = mlxsw_core_params_register(mlxsw_core);
2233 		if (err)
2234 			goto err_register_params;
2235 	}
2236 
2237 	err = mlxsw_core_fw_rev_validate(mlxsw_core, mlxsw_bus_info, mlxsw_driver->fw_req_rev,
2238 					 mlxsw_driver->fw_filename);
2239 	if (err)
2240 		goto err_fw_rev_validate;
2241 
2242 	err = mlxsw_linecards_init(mlxsw_core, mlxsw_bus_info);
2243 	if (err)
2244 		goto err_linecards_init;
2245 
2246 	err = mlxsw_core_health_init(mlxsw_core);
2247 	if (err)
2248 		goto err_health_init;
2249 
2250 	err = mlxsw_hwmon_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->hwmon);
2251 	if (err)
2252 		goto err_hwmon_init;
2253 
2254 	err = mlxsw_thermal_init(mlxsw_core, mlxsw_bus_info,
2255 				 &mlxsw_core->thermal);
2256 	if (err)
2257 		goto err_thermal_init;
2258 
2259 	err = mlxsw_env_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->env);
2260 	if (err)
2261 		goto err_env_init;
2262 
2263 	if (mlxsw_driver->init) {
2264 		err = mlxsw_driver->init(mlxsw_core, mlxsw_bus_info, extack);
2265 		if (err)
2266 			goto err_driver_init;
2267 	}
2268 
2269 	if (!reload)
2270 		devl_unlock(devlink);
2271 	return 0;
2272 
2273 err_driver_init:
2274 	mlxsw_env_fini(mlxsw_core->env);
2275 err_env_init:
2276 	mlxsw_thermal_fini(mlxsw_core->thermal);
2277 err_thermal_init:
2278 	mlxsw_hwmon_fini(mlxsw_core->hwmon);
2279 err_hwmon_init:
2280 	mlxsw_core_health_fini(mlxsw_core);
2281 err_health_init:
2282 	mlxsw_linecards_fini(mlxsw_core);
2283 err_linecards_init:
2284 err_fw_rev_validate:
2285 	if (!reload)
2286 		mlxsw_core_params_unregister(mlxsw_core);
2287 err_register_params:
2288 	mlxsw_emad_fini(mlxsw_core);
2289 err_emad_init:
2290 err_trap_groups_set:
2291 	kfree(mlxsw_core->lag.mapping);
2292 err_alloc_lag_mapping:
2293 	mlxsw_ports_fini(mlxsw_core, reload);
2294 err_ports_init:
2295 	if (!reload)
2296 		devl_resources_unregister(devlink);
2297 err_register_resources:
2298 	mlxsw_bus->fini(bus_priv);
2299 err_bus_init:
2300 	mlxsw_core_irq_event_handler_fini(mlxsw_core);
2301 	if (!reload) {
2302 		devl_unregister(devlink);
2303 		devl_unlock(devlink);
2304 		devlink_free(devlink);
2305 	}
2306 err_devlink_alloc:
2307 	return err;
2308 }
2309 
2310 int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
2311 				   const struct mlxsw_bus *mlxsw_bus,
2312 				   void *bus_priv, bool reload,
2313 				   struct devlink *devlink,
2314 				   struct netlink_ext_ack *extack)
2315 {
2316 	bool called_again = false;
2317 	int err;
2318 
2319 again:
2320 	err = __mlxsw_core_bus_device_register(mlxsw_bus_info, mlxsw_bus,
2321 					       bus_priv, reload,
2322 					       devlink, extack);
2323 	/* -EAGAIN is returned in case the FW was updated. FW needs
2324 	 * a reset, so lets try to call __mlxsw_core_bus_device_register()
2325 	 * again.
2326 	 */
2327 	if (err == -EAGAIN && !called_again) {
2328 		called_again = true;
2329 		goto again;
2330 	}
2331 
2332 	return err;
2333 }
2334 EXPORT_SYMBOL(mlxsw_core_bus_device_register);
2335 
2336 void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core,
2337 				      bool reload)
2338 {
2339 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
2340 
2341 	if (!reload)
2342 		devl_lock(devlink);
2343 
2344 	if (devlink_is_reload_failed(devlink)) {
2345 		if (!reload)
2346 			/* Only the parts that were not de-initialized in the
2347 			 * failed reload attempt need to be de-initialized.
2348 			 */
2349 			goto reload_fail_deinit;
2350 		else
2351 			return;
2352 	}
2353 
2354 	if (mlxsw_core->driver->fini)
2355 		mlxsw_core->driver->fini(mlxsw_core);
2356 	mlxsw_env_fini(mlxsw_core->env);
2357 	mlxsw_thermal_fini(mlxsw_core->thermal);
2358 	mlxsw_hwmon_fini(mlxsw_core->hwmon);
2359 	mlxsw_core_health_fini(mlxsw_core);
2360 	mlxsw_linecards_fini(mlxsw_core);
2361 	if (!reload)
2362 		mlxsw_core_params_unregister(mlxsw_core);
2363 	mlxsw_emad_fini(mlxsw_core);
2364 	kfree(mlxsw_core->lag.mapping);
2365 	mlxsw_ports_fini(mlxsw_core, reload);
2366 	if (!reload)
2367 		devl_resources_unregister(devlink);
2368 	mlxsw_core->bus->fini(mlxsw_core->bus_priv);
2369 	mlxsw_core_irq_event_handler_fini(mlxsw_core);
2370 	if (!reload) {
2371 		devl_unregister(devlink);
2372 		devl_unlock(devlink);
2373 		devlink_free(devlink);
2374 	}
2375 
2376 	return;
2377 
2378 reload_fail_deinit:
2379 	mlxsw_core_params_unregister(mlxsw_core);
2380 	devl_resources_unregister(devlink);
2381 	devl_unregister(devlink);
2382 	devl_unlock(devlink);
2383 	devlink_free(devlink);
2384 }
2385 EXPORT_SYMBOL(mlxsw_core_bus_device_unregister);
2386 
2387 bool mlxsw_core_skb_transmit_busy(struct mlxsw_core *mlxsw_core,
2388 				  const struct mlxsw_tx_info *tx_info)
2389 {
2390 	return mlxsw_core->bus->skb_transmit_busy(mlxsw_core->bus_priv,
2391 						  tx_info);
2392 }
2393 EXPORT_SYMBOL(mlxsw_core_skb_transmit_busy);
2394 
2395 int mlxsw_core_skb_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
2396 			    const struct mlxsw_tx_info *tx_info)
2397 {
2398 	return mlxsw_core->bus->skb_transmit(mlxsw_core->bus_priv, skb,
2399 					     tx_info);
2400 }
2401 EXPORT_SYMBOL(mlxsw_core_skb_transmit);
2402 
2403 void mlxsw_core_ptp_transmitted(struct mlxsw_core *mlxsw_core,
2404 				struct sk_buff *skb, u16 local_port)
2405 {
2406 	if (mlxsw_core->driver->ptp_transmitted)
2407 		mlxsw_core->driver->ptp_transmitted(mlxsw_core, skb,
2408 						    local_port);
2409 }
2410 EXPORT_SYMBOL(mlxsw_core_ptp_transmitted);
2411 
2412 static bool __is_rx_listener_equal(const struct mlxsw_rx_listener *rxl_a,
2413 				   const struct mlxsw_rx_listener *rxl_b)
2414 {
2415 	return (rxl_a->func == rxl_b->func &&
2416 		rxl_a->local_port == rxl_b->local_port &&
2417 		rxl_a->trap_id == rxl_b->trap_id &&
2418 		rxl_a->mirror_reason == rxl_b->mirror_reason);
2419 }
2420 
2421 static struct mlxsw_rx_listener_item *
2422 __find_rx_listener_item(struct mlxsw_core *mlxsw_core,
2423 			const struct mlxsw_rx_listener *rxl)
2424 {
2425 	struct mlxsw_rx_listener_item *rxl_item;
2426 
2427 	list_for_each_entry(rxl_item, &mlxsw_core->rx_listener_list, list) {
2428 		if (__is_rx_listener_equal(&rxl_item->rxl, rxl))
2429 			return rxl_item;
2430 	}
2431 	return NULL;
2432 }
2433 
2434 int mlxsw_core_rx_listener_register(struct mlxsw_core *mlxsw_core,
2435 				    const struct mlxsw_rx_listener *rxl,
2436 				    void *priv, bool enabled)
2437 {
2438 	struct mlxsw_rx_listener_item *rxl_item;
2439 
2440 	rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
2441 	if (rxl_item)
2442 		return -EEXIST;
2443 	rxl_item = kmalloc(sizeof(*rxl_item), GFP_KERNEL);
2444 	if (!rxl_item)
2445 		return -ENOMEM;
2446 	rxl_item->rxl = *rxl;
2447 	rxl_item->priv = priv;
2448 	rxl_item->enabled = enabled;
2449 
2450 	list_add_rcu(&rxl_item->list, &mlxsw_core->rx_listener_list);
2451 	return 0;
2452 }
2453 EXPORT_SYMBOL(mlxsw_core_rx_listener_register);
2454 
2455 void mlxsw_core_rx_listener_unregister(struct mlxsw_core *mlxsw_core,
2456 				       const struct mlxsw_rx_listener *rxl)
2457 {
2458 	struct mlxsw_rx_listener_item *rxl_item;
2459 
2460 	rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
2461 	if (!rxl_item)
2462 		return;
2463 	list_del_rcu(&rxl_item->list);
2464 	synchronize_rcu();
2465 	kfree(rxl_item);
2466 }
2467 EXPORT_SYMBOL(mlxsw_core_rx_listener_unregister);
2468 
2469 static void
2470 mlxsw_core_rx_listener_state_set(struct mlxsw_core *mlxsw_core,
2471 				 const struct mlxsw_rx_listener *rxl,
2472 				 bool enabled)
2473 {
2474 	struct mlxsw_rx_listener_item *rxl_item;
2475 
2476 	rxl_item = __find_rx_listener_item(mlxsw_core, rxl);
2477 	if (WARN_ON(!rxl_item))
2478 		return;
2479 	rxl_item->enabled = enabled;
2480 }
2481 
2482 static void mlxsw_core_event_listener_func(struct sk_buff *skb, u16 local_port,
2483 					   void *priv)
2484 {
2485 	struct mlxsw_event_listener_item *event_listener_item = priv;
2486 	struct mlxsw_core *mlxsw_core;
2487 	struct mlxsw_reg_info reg;
2488 	char *payload;
2489 	char *reg_tlv;
2490 	char *op_tlv;
2491 
2492 	mlxsw_core = event_listener_item->mlxsw_core;
2493 	trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), true, 0,
2494 			    skb->data, skb->len);
2495 
2496 	mlxsw_emad_tlv_parse(skb);
2497 	op_tlv = mlxsw_emad_op_tlv(skb);
2498 	reg_tlv = mlxsw_emad_reg_tlv(skb);
2499 
2500 	reg.id = mlxsw_emad_op_tlv_register_id_get(op_tlv);
2501 	reg.len = (mlxsw_emad_reg_tlv_len_get(reg_tlv) - 1) * sizeof(u32);
2502 	payload = mlxsw_emad_reg_payload(reg_tlv);
2503 	event_listener_item->el.func(&reg, payload, event_listener_item->priv);
2504 	dev_kfree_skb(skb);
2505 }
2506 
2507 static bool __is_event_listener_equal(const struct mlxsw_event_listener *el_a,
2508 				      const struct mlxsw_event_listener *el_b)
2509 {
2510 	return (el_a->func == el_b->func &&
2511 		el_a->trap_id == el_b->trap_id);
2512 }
2513 
2514 static struct mlxsw_event_listener_item *
2515 __find_event_listener_item(struct mlxsw_core *mlxsw_core,
2516 			   const struct mlxsw_event_listener *el)
2517 {
2518 	struct mlxsw_event_listener_item *el_item;
2519 
2520 	list_for_each_entry(el_item, &mlxsw_core->event_listener_list, list) {
2521 		if (__is_event_listener_equal(&el_item->el, el))
2522 			return el_item;
2523 	}
2524 	return NULL;
2525 }
2526 
2527 int mlxsw_core_event_listener_register(struct mlxsw_core *mlxsw_core,
2528 				       const struct mlxsw_event_listener *el,
2529 				       void *priv)
2530 {
2531 	int err;
2532 	struct mlxsw_event_listener_item *el_item;
2533 	const struct mlxsw_rx_listener rxl = {
2534 		.func = mlxsw_core_event_listener_func,
2535 		.local_port = MLXSW_PORT_DONT_CARE,
2536 		.trap_id = el->trap_id,
2537 	};
2538 
2539 	el_item = __find_event_listener_item(mlxsw_core, el);
2540 	if (el_item)
2541 		return -EEXIST;
2542 	el_item = kmalloc(sizeof(*el_item), GFP_KERNEL);
2543 	if (!el_item)
2544 		return -ENOMEM;
2545 	el_item->mlxsw_core = mlxsw_core;
2546 	el_item->el = *el;
2547 	el_item->priv = priv;
2548 
2549 	err = mlxsw_core_rx_listener_register(mlxsw_core, &rxl, el_item, true);
2550 	if (err)
2551 		goto err_rx_listener_register;
2552 
2553 	/* No reason to save item if we did not manage to register an RX
2554 	 * listener for it.
2555 	 */
2556 	list_add_rcu(&el_item->list, &mlxsw_core->event_listener_list);
2557 
2558 	return 0;
2559 
2560 err_rx_listener_register:
2561 	kfree(el_item);
2562 	return err;
2563 }
2564 EXPORT_SYMBOL(mlxsw_core_event_listener_register);
2565 
2566 void mlxsw_core_event_listener_unregister(struct mlxsw_core *mlxsw_core,
2567 					  const struct mlxsw_event_listener *el)
2568 {
2569 	struct mlxsw_event_listener_item *el_item;
2570 	const struct mlxsw_rx_listener rxl = {
2571 		.func = mlxsw_core_event_listener_func,
2572 		.local_port = MLXSW_PORT_DONT_CARE,
2573 		.trap_id = el->trap_id,
2574 	};
2575 
2576 	el_item = __find_event_listener_item(mlxsw_core, el);
2577 	if (!el_item)
2578 		return;
2579 	mlxsw_core_rx_listener_unregister(mlxsw_core, &rxl);
2580 	list_del(&el_item->list);
2581 	kfree(el_item);
2582 }
2583 EXPORT_SYMBOL(mlxsw_core_event_listener_unregister);
2584 
2585 static int mlxsw_core_listener_register(struct mlxsw_core *mlxsw_core,
2586 					const struct mlxsw_listener *listener,
2587 					void *priv, bool enabled)
2588 {
2589 	if (listener->is_event) {
2590 		WARN_ON(!enabled);
2591 		return mlxsw_core_event_listener_register(mlxsw_core,
2592 						&listener->event_listener,
2593 						priv);
2594 	} else {
2595 		return mlxsw_core_rx_listener_register(mlxsw_core,
2596 						&listener->rx_listener,
2597 						priv, enabled);
2598 	}
2599 }
2600 
2601 static void mlxsw_core_listener_unregister(struct mlxsw_core *mlxsw_core,
2602 				      const struct mlxsw_listener *listener,
2603 				      void *priv)
2604 {
2605 	if (listener->is_event)
2606 		mlxsw_core_event_listener_unregister(mlxsw_core,
2607 						     &listener->event_listener);
2608 	else
2609 		mlxsw_core_rx_listener_unregister(mlxsw_core,
2610 						  &listener->rx_listener);
2611 }
2612 
2613 int mlxsw_core_trap_register(struct mlxsw_core *mlxsw_core,
2614 			     const struct mlxsw_listener *listener, void *priv)
2615 {
2616 	enum mlxsw_reg_htgt_trap_group trap_group;
2617 	enum mlxsw_reg_hpkt_action action;
2618 	char hpkt_pl[MLXSW_REG_HPKT_LEN];
2619 	int err;
2620 
2621 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
2622 		return 0;
2623 
2624 	err = mlxsw_core_listener_register(mlxsw_core, listener, priv,
2625 					   listener->enabled_on_register);
2626 	if (err)
2627 		return err;
2628 
2629 	action = listener->enabled_on_register ? listener->en_action :
2630 						 listener->dis_action;
2631 	trap_group = listener->enabled_on_register ? listener->en_trap_group :
2632 						     listener->dis_trap_group;
2633 	mlxsw_reg_hpkt_pack(hpkt_pl, action, listener->trap_id,
2634 			    trap_group, listener->is_ctrl);
2635 	err = mlxsw_reg_write(mlxsw_core,  MLXSW_REG(hpkt), hpkt_pl);
2636 	if (err)
2637 		goto err_trap_set;
2638 
2639 	return 0;
2640 
2641 err_trap_set:
2642 	mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
2643 	return err;
2644 }
2645 EXPORT_SYMBOL(mlxsw_core_trap_register);
2646 
2647 void mlxsw_core_trap_unregister(struct mlxsw_core *mlxsw_core,
2648 				const struct mlxsw_listener *listener,
2649 				void *priv)
2650 {
2651 	char hpkt_pl[MLXSW_REG_HPKT_LEN];
2652 
2653 	if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
2654 		return;
2655 
2656 	if (!listener->is_event) {
2657 		mlxsw_reg_hpkt_pack(hpkt_pl, listener->dis_action,
2658 				    listener->trap_id, listener->dis_trap_group,
2659 				    listener->is_ctrl);
2660 		mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
2661 	}
2662 
2663 	mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
2664 }
2665 EXPORT_SYMBOL(mlxsw_core_trap_unregister);
2666 
2667 int mlxsw_core_traps_register(struct mlxsw_core *mlxsw_core,
2668 			      const struct mlxsw_listener *listeners,
2669 			      size_t listeners_count, void *priv)
2670 {
2671 	int i, err;
2672 
2673 	for (i = 0; i < listeners_count; i++) {
2674 		err = mlxsw_core_trap_register(mlxsw_core,
2675 					       &listeners[i],
2676 					       priv);
2677 		if (err)
2678 			goto err_listener_register;
2679 	}
2680 	return 0;
2681 
2682 err_listener_register:
2683 	for (i--; i >= 0; i--) {
2684 		mlxsw_core_trap_unregister(mlxsw_core,
2685 					   &listeners[i],
2686 					   priv);
2687 	}
2688 	return err;
2689 }
2690 EXPORT_SYMBOL(mlxsw_core_traps_register);
2691 
2692 void mlxsw_core_traps_unregister(struct mlxsw_core *mlxsw_core,
2693 				 const struct mlxsw_listener *listeners,
2694 				 size_t listeners_count, void *priv)
2695 {
2696 	int i;
2697 
2698 	for (i = 0; i < listeners_count; i++) {
2699 		mlxsw_core_trap_unregister(mlxsw_core,
2700 					   &listeners[i],
2701 					   priv);
2702 	}
2703 }
2704 EXPORT_SYMBOL(mlxsw_core_traps_unregister);
2705 
2706 int mlxsw_core_trap_state_set(struct mlxsw_core *mlxsw_core,
2707 			      const struct mlxsw_listener *listener,
2708 			      bool enabled)
2709 {
2710 	enum mlxsw_reg_htgt_trap_group trap_group;
2711 	enum mlxsw_reg_hpkt_action action;
2712 	char hpkt_pl[MLXSW_REG_HPKT_LEN];
2713 	int err;
2714 
2715 	/* Not supported for event listener */
2716 	if (WARN_ON(listener->is_event))
2717 		return -EINVAL;
2718 
2719 	action = enabled ? listener->en_action : listener->dis_action;
2720 	trap_group = enabled ? listener->en_trap_group :
2721 			       listener->dis_trap_group;
2722 	mlxsw_reg_hpkt_pack(hpkt_pl, action, listener->trap_id,
2723 			    trap_group, listener->is_ctrl);
2724 	err = mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
2725 	if (err)
2726 		return err;
2727 
2728 	mlxsw_core_rx_listener_state_set(mlxsw_core, &listener->rx_listener,
2729 					 enabled);
2730 	return 0;
2731 }
2732 EXPORT_SYMBOL(mlxsw_core_trap_state_set);
2733 
2734 static u64 mlxsw_core_tid_get(struct mlxsw_core *mlxsw_core)
2735 {
2736 	return atomic64_inc_return(&mlxsw_core->emad.tid);
2737 }
2738 
2739 static int mlxsw_core_reg_access_emad(struct mlxsw_core *mlxsw_core,
2740 				      const struct mlxsw_reg_info *reg,
2741 				      char *payload,
2742 				      enum mlxsw_core_reg_access_type type,
2743 				      struct list_head *bulk_list,
2744 				      mlxsw_reg_trans_cb_t *cb,
2745 				      unsigned long cb_priv)
2746 {
2747 	u64 tid = mlxsw_core_tid_get(mlxsw_core);
2748 	struct mlxsw_reg_trans *trans;
2749 	int err;
2750 
2751 	trans = kzalloc(sizeof(*trans), GFP_KERNEL);
2752 	if (!trans)
2753 		return -ENOMEM;
2754 
2755 	err = mlxsw_emad_reg_access(mlxsw_core, reg, payload, type, trans,
2756 				    bulk_list, cb, cb_priv, tid);
2757 	if (err) {
2758 		kfree_rcu(trans, rcu);
2759 		return err;
2760 	}
2761 	return 0;
2762 }
2763 
2764 int mlxsw_reg_trans_query(struct mlxsw_core *mlxsw_core,
2765 			  const struct mlxsw_reg_info *reg, char *payload,
2766 			  struct list_head *bulk_list,
2767 			  mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
2768 {
2769 	return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
2770 					  MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
2771 					  bulk_list, cb, cb_priv);
2772 }
2773 EXPORT_SYMBOL(mlxsw_reg_trans_query);
2774 
2775 int mlxsw_reg_trans_write(struct mlxsw_core *mlxsw_core,
2776 			  const struct mlxsw_reg_info *reg, char *payload,
2777 			  struct list_head *bulk_list,
2778 			  mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
2779 {
2780 	return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
2781 					  MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
2782 					  bulk_list, cb, cb_priv);
2783 }
2784 EXPORT_SYMBOL(mlxsw_reg_trans_write);
2785 
2786 #define MLXSW_REG_TRANS_ERR_STRING_SIZE	256
2787 
2788 static int mlxsw_reg_trans_wait(struct mlxsw_reg_trans *trans)
2789 {
2790 	char err_string[MLXSW_REG_TRANS_ERR_STRING_SIZE];
2791 	struct mlxsw_core *mlxsw_core = trans->core;
2792 	int err;
2793 
2794 	wait_for_completion(&trans->completion);
2795 	cancel_delayed_work_sync(&trans->timeout_dw);
2796 	err = trans->err;
2797 
2798 	if (trans->retries)
2799 		dev_warn(mlxsw_core->bus_info->dev, "EMAD retries (%d/%d) (tid=%llx)\n",
2800 			 trans->retries, MLXSW_EMAD_MAX_RETRY, trans->tid);
2801 	if (err) {
2802 		dev_err(mlxsw_core->bus_info->dev, "EMAD reg access failed (tid=%llx,reg_id=%x(%s),type=%s,status=%x(%s))\n",
2803 			trans->tid, trans->reg->id,
2804 			mlxsw_reg_id_str(trans->reg->id),
2805 			mlxsw_core_reg_access_type_str(trans->type),
2806 			trans->emad_status,
2807 			mlxsw_emad_op_tlv_status_str(trans->emad_status));
2808 
2809 		snprintf(err_string, MLXSW_REG_TRANS_ERR_STRING_SIZE,
2810 			 "(tid=%llx,reg_id=%x(%s)) %s (%s)\n", trans->tid,
2811 			 trans->reg->id, mlxsw_reg_id_str(trans->reg->id),
2812 			 mlxsw_emad_op_tlv_status_str(trans->emad_status),
2813 			 trans->emad_err_string ? trans->emad_err_string : "");
2814 
2815 		trace_devlink_hwerr(priv_to_devlink(mlxsw_core),
2816 				    trans->emad_status, err_string);
2817 
2818 		kfree(trans->emad_err_string);
2819 	}
2820 
2821 	list_del(&trans->bulk_list);
2822 	kfree_rcu(trans, rcu);
2823 	return err;
2824 }
2825 
2826 int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list)
2827 {
2828 	struct mlxsw_reg_trans *trans;
2829 	struct mlxsw_reg_trans *tmp;
2830 	int sum_err = 0;
2831 	int err;
2832 
2833 	list_for_each_entry_safe(trans, tmp, bulk_list, bulk_list) {
2834 		err = mlxsw_reg_trans_wait(trans);
2835 		if (err && sum_err == 0)
2836 			sum_err = err; /* first error to be returned */
2837 	}
2838 	return sum_err;
2839 }
2840 EXPORT_SYMBOL(mlxsw_reg_trans_bulk_wait);
2841 
2842 struct mlxsw_core_irq_event_handler_item {
2843 	struct list_head list;
2844 	void (*cb)(struct mlxsw_core *mlxsw_core);
2845 };
2846 
2847 int mlxsw_core_irq_event_handler_register(struct mlxsw_core *mlxsw_core,
2848 					  mlxsw_irq_event_cb_t cb)
2849 {
2850 	struct mlxsw_core_irq_event_handler_item *item;
2851 
2852 	item = kzalloc(sizeof(*item), GFP_KERNEL);
2853 	if (!item)
2854 		return -ENOMEM;
2855 	item->cb = cb;
2856 	mutex_lock(&mlxsw_core->irq_event_handler_lock);
2857 	list_add_tail(&item->list, &mlxsw_core->irq_event_handler_list);
2858 	mutex_unlock(&mlxsw_core->irq_event_handler_lock);
2859 	return 0;
2860 }
2861 EXPORT_SYMBOL(mlxsw_core_irq_event_handler_register);
2862 
2863 void mlxsw_core_irq_event_handler_unregister(struct mlxsw_core *mlxsw_core,
2864 					     mlxsw_irq_event_cb_t cb)
2865 {
2866 	struct mlxsw_core_irq_event_handler_item *item, *tmp;
2867 
2868 	mutex_lock(&mlxsw_core->irq_event_handler_lock);
2869 	list_for_each_entry_safe(item, tmp,
2870 				 &mlxsw_core->irq_event_handler_list, list) {
2871 		if (item->cb == cb) {
2872 			list_del(&item->list);
2873 			kfree(item);
2874 		}
2875 	}
2876 	mutex_unlock(&mlxsw_core->irq_event_handler_lock);
2877 }
2878 EXPORT_SYMBOL(mlxsw_core_irq_event_handler_unregister);
2879 
2880 void mlxsw_core_irq_event_handlers_call(struct mlxsw_core *mlxsw_core)
2881 {
2882 	struct mlxsw_core_irq_event_handler_item *item;
2883 
2884 	mutex_lock(&mlxsw_core->irq_event_handler_lock);
2885 	list_for_each_entry(item, &mlxsw_core->irq_event_handler_list, list) {
2886 		if (item->cb)
2887 			item->cb(mlxsw_core);
2888 	}
2889 	mutex_unlock(&mlxsw_core->irq_event_handler_lock);
2890 }
2891 EXPORT_SYMBOL(mlxsw_core_irq_event_handlers_call);
2892 
2893 static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
2894 				     const struct mlxsw_reg_info *reg,
2895 				     char *payload,
2896 				     enum mlxsw_core_reg_access_type type)
2897 {
2898 	enum mlxsw_emad_op_tlv_status status;
2899 	int err, n_retry;
2900 	bool reset_ok;
2901 	char *in_mbox, *out_mbox, *tmp;
2902 
2903 	dev_dbg(mlxsw_core->bus_info->dev, "Reg cmd access (reg_id=%x(%s),type=%s)\n",
2904 		reg->id, mlxsw_reg_id_str(reg->id),
2905 		mlxsw_core_reg_access_type_str(type));
2906 
2907 	in_mbox = mlxsw_cmd_mbox_alloc();
2908 	if (!in_mbox)
2909 		return -ENOMEM;
2910 
2911 	out_mbox = mlxsw_cmd_mbox_alloc();
2912 	if (!out_mbox) {
2913 		err = -ENOMEM;
2914 		goto free_in_mbox;
2915 	}
2916 
2917 	mlxsw_emad_pack_op_tlv(in_mbox, reg, type,
2918 			       mlxsw_core_tid_get(mlxsw_core));
2919 	tmp = in_mbox + MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
2920 	mlxsw_emad_pack_reg_tlv(tmp, reg, payload);
2921 
2922 	/* There is a special treatment needed for MRSR (reset) register.
2923 	 * The command interface will return error after the command
2924 	 * is executed, so tell the lower layer to expect it
2925 	 * and cope accordingly.
2926 	 */
2927 	reset_ok = reg->id == MLXSW_REG_MRSR_ID;
2928 
2929 	n_retry = 0;
2930 retry:
2931 	err = mlxsw_cmd_access_reg(mlxsw_core, reset_ok, in_mbox, out_mbox);
2932 	if (!err) {
2933 		err = mlxsw_emad_process_status(out_mbox, &status);
2934 		if (err) {
2935 			if (err == -EAGAIN && n_retry++ < MLXSW_EMAD_MAX_RETRY)
2936 				goto retry;
2937 			dev_err(mlxsw_core->bus_info->dev, "Reg cmd access status failed (status=%x(%s))\n",
2938 				status, mlxsw_emad_op_tlv_status_str(status));
2939 		}
2940 	}
2941 
2942 	if (!err)
2943 		memcpy(payload, mlxsw_emad_reg_payload_cmd(out_mbox),
2944 		       reg->len);
2945 
2946 	mlxsw_cmd_mbox_free(out_mbox);
2947 free_in_mbox:
2948 	mlxsw_cmd_mbox_free(in_mbox);
2949 	if (err)
2950 		dev_err(mlxsw_core->bus_info->dev, "Reg cmd access failed (reg_id=%x(%s),type=%s)\n",
2951 			reg->id, mlxsw_reg_id_str(reg->id),
2952 			mlxsw_core_reg_access_type_str(type));
2953 	return err;
2954 }
2955 
2956 static void mlxsw_core_reg_access_cb(struct mlxsw_core *mlxsw_core,
2957 				     char *payload, size_t payload_len,
2958 				     unsigned long cb_priv)
2959 {
2960 	char *orig_payload = (char *) cb_priv;
2961 
2962 	memcpy(orig_payload, payload, payload_len);
2963 }
2964 
2965 static int mlxsw_core_reg_access(struct mlxsw_core *mlxsw_core,
2966 				 const struct mlxsw_reg_info *reg,
2967 				 char *payload,
2968 				 enum mlxsw_core_reg_access_type type)
2969 {
2970 	LIST_HEAD(bulk_list);
2971 	int err;
2972 
2973 	/* During initialization EMAD interface is not available to us,
2974 	 * so we default to command interface. We switch to EMAD interface
2975 	 * after setting the appropriate traps.
2976 	 */
2977 	if (!mlxsw_core->emad.use_emad)
2978 		return mlxsw_core_reg_access_cmd(mlxsw_core, reg,
2979 						 payload, type);
2980 
2981 	err = mlxsw_core_reg_access_emad(mlxsw_core, reg,
2982 					 payload, type, &bulk_list,
2983 					 mlxsw_core_reg_access_cb,
2984 					 (unsigned long) payload);
2985 	if (err)
2986 		return err;
2987 	return mlxsw_reg_trans_bulk_wait(&bulk_list);
2988 }
2989 
2990 int mlxsw_reg_query(struct mlxsw_core *mlxsw_core,
2991 		    const struct mlxsw_reg_info *reg, char *payload)
2992 {
2993 	return mlxsw_core_reg_access(mlxsw_core, reg, payload,
2994 				     MLXSW_CORE_REG_ACCESS_TYPE_QUERY);
2995 }
2996 EXPORT_SYMBOL(mlxsw_reg_query);
2997 
2998 int mlxsw_reg_write(struct mlxsw_core *mlxsw_core,
2999 		    const struct mlxsw_reg_info *reg, char *payload)
3000 {
3001 	return mlxsw_core_reg_access(mlxsw_core, reg, payload,
3002 				     MLXSW_CORE_REG_ACCESS_TYPE_WRITE);
3003 }
3004 EXPORT_SYMBOL(mlxsw_reg_write);
3005 
3006 void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
3007 			    struct mlxsw_rx_info *rx_info)
3008 {
3009 	struct mlxsw_rx_listener_item *rxl_item;
3010 	const struct mlxsw_rx_listener *rxl;
3011 	u16 local_port;
3012 	bool found = false;
3013 
3014 	if (rx_info->is_lag) {
3015 		dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: lag_id = %d, lag_port_index = 0x%x\n",
3016 				    __func__, rx_info->u.lag_id,
3017 				    rx_info->trap_id);
3018 		/* Upper layer does not care if the skb came from LAG or not,
3019 		 * so just get the local_port for the lag port and push it up.
3020 		 */
3021 		local_port = mlxsw_core_lag_mapping_get(mlxsw_core,
3022 							rx_info->u.lag_id,
3023 							rx_info->lag_port_index);
3024 	} else {
3025 		local_port = rx_info->u.sys_port;
3026 	}
3027 
3028 	dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: local_port = %d, trap_id = 0x%x\n",
3029 			    __func__, local_port, rx_info->trap_id);
3030 
3031 	if ((rx_info->trap_id >= MLXSW_TRAP_ID_MAX) ||
3032 	    (local_port >= mlxsw_core->max_ports))
3033 		goto drop;
3034 
3035 	rcu_read_lock();
3036 	list_for_each_entry_rcu(rxl_item, &mlxsw_core->rx_listener_list, list) {
3037 		rxl = &rxl_item->rxl;
3038 		if ((rxl->local_port == MLXSW_PORT_DONT_CARE ||
3039 		     rxl->local_port == local_port) &&
3040 		    rxl->trap_id == rx_info->trap_id &&
3041 		    rxl->mirror_reason == rx_info->mirror_reason) {
3042 			if (rxl_item->enabled)
3043 				found = true;
3044 			break;
3045 		}
3046 	}
3047 	if (!found) {
3048 		rcu_read_unlock();
3049 		goto drop;
3050 	}
3051 
3052 	rxl->func(skb, local_port, rxl_item->priv);
3053 	rcu_read_unlock();
3054 	return;
3055 
3056 drop:
3057 	dev_kfree_skb(skb);
3058 }
3059 EXPORT_SYMBOL(mlxsw_core_skb_receive);
3060 
3061 static int mlxsw_core_lag_mapping_index(struct mlxsw_core *mlxsw_core,
3062 					u16 lag_id, u8 port_index)
3063 {
3064 	return MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS) * lag_id +
3065 	       port_index;
3066 }
3067 
3068 void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core,
3069 				u16 lag_id, u8 port_index, u16 local_port)
3070 {
3071 	int index = mlxsw_core_lag_mapping_index(mlxsw_core,
3072 						 lag_id, port_index);
3073 
3074 	mlxsw_core->lag.mapping[index] = local_port;
3075 }
3076 EXPORT_SYMBOL(mlxsw_core_lag_mapping_set);
3077 
3078 u16 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core,
3079 			       u16 lag_id, u8 port_index)
3080 {
3081 	int index = mlxsw_core_lag_mapping_index(mlxsw_core,
3082 						 lag_id, port_index);
3083 
3084 	return mlxsw_core->lag.mapping[index];
3085 }
3086 EXPORT_SYMBOL(mlxsw_core_lag_mapping_get);
3087 
3088 void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core,
3089 				  u16 lag_id, u16 local_port)
3090 {
3091 	int i;
3092 
3093 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); i++) {
3094 		int index = mlxsw_core_lag_mapping_index(mlxsw_core,
3095 							 lag_id, i);
3096 
3097 		if (mlxsw_core->lag.mapping[index] == local_port)
3098 			mlxsw_core->lag.mapping[index] = 0;
3099 	}
3100 }
3101 EXPORT_SYMBOL(mlxsw_core_lag_mapping_clear);
3102 
3103 bool mlxsw_core_res_valid(struct mlxsw_core *mlxsw_core,
3104 			  enum mlxsw_res_id res_id)
3105 {
3106 	return mlxsw_res_valid(&mlxsw_core->res, res_id);
3107 }
3108 EXPORT_SYMBOL(mlxsw_core_res_valid);
3109 
3110 u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core,
3111 		       enum mlxsw_res_id res_id)
3112 {
3113 	return mlxsw_res_get(&mlxsw_core->res, res_id);
3114 }
3115 EXPORT_SYMBOL(mlxsw_core_res_get);
3116 
3117 static const struct devlink_port_ops mlxsw_devlink_port_ops = {
3118 	.port_split			= mlxsw_devlink_port_split,
3119 	.port_unsplit			= mlxsw_devlink_port_unsplit,
3120 };
3121 
3122 static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u16 local_port,
3123 				  enum devlink_port_flavour flavour,
3124 				  u8 slot_index, u32 port_number, bool split,
3125 				  u32 split_port_subnumber,
3126 				  bool splittable, u32 lanes,
3127 				  const unsigned char *switch_id,
3128 				  unsigned char switch_id_len)
3129 {
3130 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
3131 	struct mlxsw_core_port *mlxsw_core_port =
3132 					&mlxsw_core->ports[local_port];
3133 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
3134 	struct devlink_port_attrs attrs = {};
3135 	int err;
3136 
3137 	attrs.split = split;
3138 	attrs.lanes = lanes;
3139 	attrs.splittable = splittable;
3140 	attrs.flavour = flavour;
3141 	attrs.phys.port_number = port_number;
3142 	attrs.phys.split_subport_number = split_port_subnumber;
3143 	memcpy(attrs.switch_id.id, switch_id, switch_id_len);
3144 	attrs.switch_id.id_len = switch_id_len;
3145 	mlxsw_core_port->local_port = local_port;
3146 	devlink_port_attrs_set(devlink_port, &attrs);
3147 	if (slot_index) {
3148 		struct mlxsw_linecard *linecard;
3149 
3150 		linecard = mlxsw_linecard_get(mlxsw_core->linecards,
3151 					      slot_index);
3152 		mlxsw_core_port->linecard = linecard;
3153 		devlink_port_linecard_set(devlink_port,
3154 					  linecard->devlink_linecard);
3155 	}
3156 	err = devl_port_register_with_ops(devlink, devlink_port, local_port,
3157 					  &mlxsw_devlink_port_ops);
3158 	if (err)
3159 		memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
3160 	return err;
3161 }
3162 
3163 static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u16 local_port)
3164 {
3165 	struct mlxsw_core_port *mlxsw_core_port =
3166 					&mlxsw_core->ports[local_port];
3167 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
3168 
3169 	devl_port_unregister(devlink_port);
3170 	memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
3171 }
3172 
3173 int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u16 local_port,
3174 			 u8 slot_index, u32 port_number, bool split,
3175 			 u32 split_port_subnumber,
3176 			 bool splittable, u32 lanes,
3177 			 const unsigned char *switch_id,
3178 			 unsigned char switch_id_len)
3179 {
3180 	int err;
3181 
3182 	err = __mlxsw_core_port_init(mlxsw_core, local_port,
3183 				     DEVLINK_PORT_FLAVOUR_PHYSICAL, slot_index,
3184 				     port_number, split, split_port_subnumber,
3185 				     splittable, lanes,
3186 				     switch_id, switch_id_len);
3187 	if (err)
3188 		return err;
3189 
3190 	atomic_inc(&mlxsw_core->active_ports_count);
3191 	return 0;
3192 }
3193 EXPORT_SYMBOL(mlxsw_core_port_init);
3194 
3195 void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u16 local_port)
3196 {
3197 	atomic_dec(&mlxsw_core->active_ports_count);
3198 
3199 	__mlxsw_core_port_fini(mlxsw_core, local_port);
3200 }
3201 EXPORT_SYMBOL(mlxsw_core_port_fini);
3202 
3203 int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core,
3204 			     void *port_driver_priv,
3205 			     const unsigned char *switch_id,
3206 			     unsigned char switch_id_len)
3207 {
3208 	struct mlxsw_core_port *mlxsw_core_port =
3209 				&mlxsw_core->ports[MLXSW_PORT_CPU_PORT];
3210 	int err;
3211 
3212 	err = __mlxsw_core_port_init(mlxsw_core, MLXSW_PORT_CPU_PORT,
3213 				     DEVLINK_PORT_FLAVOUR_CPU,
3214 				     0, 0, false, 0, false, 0,
3215 				     switch_id, switch_id_len);
3216 	if (err)
3217 		return err;
3218 
3219 	mlxsw_core_port->port_driver_priv = port_driver_priv;
3220 	return 0;
3221 }
3222 EXPORT_SYMBOL(mlxsw_core_cpu_port_init);
3223 
3224 void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core)
3225 {
3226 	__mlxsw_core_port_fini(mlxsw_core, MLXSW_PORT_CPU_PORT);
3227 }
3228 EXPORT_SYMBOL(mlxsw_core_cpu_port_fini);
3229 
3230 void mlxsw_core_port_netdev_link(struct mlxsw_core *mlxsw_core, u16 local_port,
3231 				 void *port_driver_priv, struct net_device *dev)
3232 {
3233 	struct mlxsw_core_port *mlxsw_core_port =
3234 					&mlxsw_core->ports[local_port];
3235 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
3236 
3237 	mlxsw_core_port->port_driver_priv = port_driver_priv;
3238 	SET_NETDEV_DEVLINK_PORT(dev, devlink_port);
3239 }
3240 EXPORT_SYMBOL(mlxsw_core_port_netdev_link);
3241 
3242 struct devlink_port *
3243 mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core,
3244 				 u16 local_port)
3245 {
3246 	struct mlxsw_core_port *mlxsw_core_port =
3247 					&mlxsw_core->ports[local_port];
3248 	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
3249 
3250 	return devlink_port;
3251 }
3252 EXPORT_SYMBOL(mlxsw_core_port_devlink_port_get);
3253 
3254 struct mlxsw_linecard *
3255 mlxsw_core_port_linecard_get(struct mlxsw_core *mlxsw_core,
3256 			     u16 local_port)
3257 {
3258 	struct mlxsw_core_port *mlxsw_core_port =
3259 					&mlxsw_core->ports[local_port];
3260 
3261 	return mlxsw_core_port->linecard;
3262 }
3263 
3264 void mlxsw_core_ports_remove_selected(struct mlxsw_core *mlxsw_core,
3265 				      bool (*selector)(void *priv, u16 local_port),
3266 				      void *priv)
3267 {
3268 	if (WARN_ON_ONCE(!mlxsw_core->driver->ports_remove_selected))
3269 		return;
3270 	mlxsw_core->driver->ports_remove_selected(mlxsw_core, selector, priv);
3271 }
3272 
3273 struct mlxsw_env *mlxsw_core_env(const struct mlxsw_core *mlxsw_core)
3274 {
3275 	return mlxsw_core->env;
3276 }
3277 
3278 static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core,
3279 				    const char *buf, size_t size)
3280 {
3281 	__be32 *m = (__be32 *) buf;
3282 	int i;
3283 	int count = size / sizeof(__be32);
3284 
3285 	for (i = count - 1; i >= 0; i--)
3286 		if (m[i])
3287 			break;
3288 	i++;
3289 	count = i ? i : 1;
3290 	for (i = 0; i < count; i += 4)
3291 		dev_dbg(mlxsw_core->bus_info->dev, "%04x - %08x %08x %08x %08x\n",
3292 			i * 4, be32_to_cpu(m[i]), be32_to_cpu(m[i + 1]),
3293 			be32_to_cpu(m[i + 2]), be32_to_cpu(m[i + 3]));
3294 }
3295 
3296 int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
3297 		   u32 in_mod, bool out_mbox_direct, bool reset_ok,
3298 		   char *in_mbox, size_t in_mbox_size,
3299 		   char *out_mbox, size_t out_mbox_size)
3300 {
3301 	u8 status;
3302 	int err;
3303 
3304 	BUG_ON(in_mbox_size % sizeof(u32) || out_mbox_size % sizeof(u32));
3305 	if (!mlxsw_core->bus->cmd_exec)
3306 		return -EOPNOTSUPP;
3307 
3308 	dev_dbg(mlxsw_core->bus_info->dev, "Cmd exec (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
3309 		opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod, in_mod);
3310 	if (in_mbox) {
3311 		dev_dbg(mlxsw_core->bus_info->dev, "Input mailbox:\n");
3312 		mlxsw_core_buf_dump_dbg(mlxsw_core, in_mbox, in_mbox_size);
3313 	}
3314 
3315 	err = mlxsw_core->bus->cmd_exec(mlxsw_core->bus_priv, opcode,
3316 					opcode_mod, in_mod, out_mbox_direct,
3317 					in_mbox, in_mbox_size,
3318 					out_mbox, out_mbox_size, &status);
3319 
3320 	if (!err && out_mbox) {
3321 		dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n");
3322 		mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size);
3323 	}
3324 
3325 	if (reset_ok && err == -EIO &&
3326 	    status == MLXSW_CMD_STATUS_RUNNING_RESET) {
3327 		err = 0;
3328 	} else if (err == -EIO && status != MLXSW_CMD_STATUS_OK) {
3329 		dev_err(mlxsw_core->bus_info->dev, "Cmd exec failed (opcode=%x(%s),opcode_mod=%x,in_mod=%x,status=%x(%s))\n",
3330 			opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
3331 			in_mod, status, mlxsw_cmd_status_str(status));
3332 	} else if (err == -ETIMEDOUT) {
3333 		dev_err(mlxsw_core->bus_info->dev, "Cmd exec timed-out (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
3334 			opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
3335 			in_mod);
3336 	}
3337 
3338 	return err;
3339 }
3340 EXPORT_SYMBOL(mlxsw_cmd_exec);
3341 
3342 int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay)
3343 {
3344 	return queue_delayed_work(mlxsw_wq, dwork, delay);
3345 }
3346 EXPORT_SYMBOL(mlxsw_core_schedule_dw);
3347 
3348 bool mlxsw_core_schedule_work(struct work_struct *work)
3349 {
3350 	return queue_work(mlxsw_owq, work);
3351 }
3352 EXPORT_SYMBOL(mlxsw_core_schedule_work);
3353 
3354 void mlxsw_core_flush_owq(void)
3355 {
3356 	flush_workqueue(mlxsw_owq);
3357 }
3358 EXPORT_SYMBOL(mlxsw_core_flush_owq);
3359 
3360 int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core,
3361 			     const struct mlxsw_config_profile *profile,
3362 			     u64 *p_single_size, u64 *p_double_size,
3363 			     u64 *p_linear_size)
3364 {
3365 	struct mlxsw_driver *driver = mlxsw_core->driver;
3366 
3367 	if (!driver->kvd_sizes_get)
3368 		return -EINVAL;
3369 
3370 	return driver->kvd_sizes_get(mlxsw_core, profile,
3371 				     p_single_size, p_double_size,
3372 				     p_linear_size);
3373 }
3374 EXPORT_SYMBOL(mlxsw_core_kvd_sizes_get);
3375 
3376 int mlxsw_core_resources_query(struct mlxsw_core *mlxsw_core, char *mbox,
3377 			       struct mlxsw_res *res)
3378 {
3379 	int index, i;
3380 	u64 data;
3381 	u16 id;
3382 	int err;
3383 
3384 	mlxsw_cmd_mbox_zero(mbox);
3385 
3386 	for (index = 0; index < MLXSW_CMD_QUERY_RESOURCES_MAX_QUERIES;
3387 	     index++) {
3388 		err = mlxsw_cmd_query_resources(mlxsw_core, mbox, index);
3389 		if (err)
3390 			return err;
3391 
3392 		for (i = 0; i < MLXSW_CMD_QUERY_RESOURCES_PER_QUERY; i++) {
3393 			id = mlxsw_cmd_mbox_query_resource_id_get(mbox, i);
3394 			data = mlxsw_cmd_mbox_query_resource_data_get(mbox, i);
3395 
3396 			if (id == MLXSW_CMD_QUERY_RESOURCES_TABLE_END_ID)
3397 				return 0;
3398 
3399 			mlxsw_res_parse(res, id, data);
3400 		}
3401 	}
3402 
3403 	/* If after MLXSW_RESOURCES_QUERY_MAX_QUERIES we still didn't get
3404 	 * MLXSW_RESOURCES_TABLE_END_ID, something went bad in the FW.
3405 	 */
3406 	return -EIO;
3407 }
3408 EXPORT_SYMBOL(mlxsw_core_resources_query);
3409 
3410 u32 mlxsw_core_read_frc_h(struct mlxsw_core *mlxsw_core)
3411 {
3412 	return mlxsw_core->bus->read_frc_h(mlxsw_core->bus_priv);
3413 }
3414 EXPORT_SYMBOL(mlxsw_core_read_frc_h);
3415 
3416 u32 mlxsw_core_read_frc_l(struct mlxsw_core *mlxsw_core)
3417 {
3418 	return mlxsw_core->bus->read_frc_l(mlxsw_core->bus_priv);
3419 }
3420 EXPORT_SYMBOL(mlxsw_core_read_frc_l);
3421 
3422 u32 mlxsw_core_read_utc_sec(struct mlxsw_core *mlxsw_core)
3423 {
3424 	return mlxsw_core->bus->read_utc_sec(mlxsw_core->bus_priv);
3425 }
3426 EXPORT_SYMBOL(mlxsw_core_read_utc_sec);
3427 
3428 u32 mlxsw_core_read_utc_nsec(struct mlxsw_core *mlxsw_core)
3429 {
3430 	return mlxsw_core->bus->read_utc_nsec(mlxsw_core->bus_priv);
3431 }
3432 EXPORT_SYMBOL(mlxsw_core_read_utc_nsec);
3433 
3434 bool mlxsw_core_sdq_supports_cqe_v2(struct mlxsw_core *mlxsw_core)
3435 {
3436 	return mlxsw_core->driver->sdq_supports_cqe_v2;
3437 }
3438 EXPORT_SYMBOL(mlxsw_core_sdq_supports_cqe_v2);
3439 
3440 static int __init mlxsw_core_module_init(void)
3441 {
3442 	int err;
3443 
3444 	err = mlxsw_linecard_driver_register();
3445 	if (err)
3446 		return err;
3447 
3448 	mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, 0, 0);
3449 	if (!mlxsw_wq) {
3450 		err = -ENOMEM;
3451 		goto err_alloc_workqueue;
3452 	}
3453 	mlxsw_owq = alloc_ordered_workqueue("%s_ordered", 0,
3454 					    mlxsw_core_driver_name);
3455 	if (!mlxsw_owq) {
3456 		err = -ENOMEM;
3457 		goto err_alloc_ordered_workqueue;
3458 	}
3459 	return 0;
3460 
3461 err_alloc_ordered_workqueue:
3462 	destroy_workqueue(mlxsw_wq);
3463 err_alloc_workqueue:
3464 	mlxsw_linecard_driver_unregister();
3465 	return err;
3466 }
3467 
3468 static void __exit mlxsw_core_module_exit(void)
3469 {
3470 	destroy_workqueue(mlxsw_owq);
3471 	destroy_workqueue(mlxsw_wq);
3472 	mlxsw_linecard_driver_unregister();
3473 }
3474 
3475 module_init(mlxsw_core_module_init);
3476 module_exit(mlxsw_core_module_exit);
3477 
3478 MODULE_LICENSE("Dual BSD/GPL");
3479 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
3480 MODULE_DESCRIPTION("Mellanox switch device core driver");
3481