1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/types.h>
6 #include <linux/dcbnl.h>
7 #include <linux/if_ether.h>
8 #include <linux/list.h>
9 #include <linux/netlink.h>
10 
11 #include "spectrum.h"
12 #include "core.h"
13 #include "port.h"
14 #include "reg.h"
15 
16 struct mlxsw_sp_sb_pr {
17 	enum mlxsw_reg_sbpr_mode mode;
18 	u32 size;
19 	u8 freeze_mode:1,
20 	   freeze_size:1;
21 };
22 
23 struct mlxsw_cp_sb_occ {
24 	u32 cur;
25 	u32 max;
26 };
27 
28 struct mlxsw_sp_sb_cm {
29 	u32 min_buff;
30 	u32 max_buff;
31 	u16 pool_index;
32 	struct mlxsw_cp_sb_occ occ;
33 	u8 freeze_pool:1,
34 	   freeze_thresh:1;
35 };
36 
37 #define MLXSW_SP_SB_INFI -1U
38 #define MLXSW_SP_SB_REST -2U
39 
40 struct mlxsw_sp_sb_pm {
41 	u32 min_buff;
42 	u32 max_buff;
43 	struct mlxsw_cp_sb_occ occ;
44 };
45 
46 struct mlxsw_sp_sb_mm {
47 	u32 min_buff;
48 	u32 max_buff;
49 	u16 pool_index;
50 };
51 
52 struct mlxsw_sp_sb_pool_des {
53 	enum mlxsw_reg_sbxx_dir dir;
54 	u8 pool;
55 };
56 
57 #define MLXSW_SP_SB_POOL_ING		0
58 #define MLXSW_SP_SB_POOL_EGR		4
59 #define MLXSW_SP_SB_POOL_EGR_MC		8
60 #define MLXSW_SP_SB_POOL_ING_CPU	9
61 #define MLXSW_SP_SB_POOL_EGR_CPU	10
62 
63 static const struct mlxsw_sp_sb_pool_des mlxsw_sp1_sb_pool_dess[] = {
64 	{MLXSW_REG_SBXX_DIR_INGRESS, 0},
65 	{MLXSW_REG_SBXX_DIR_INGRESS, 1},
66 	{MLXSW_REG_SBXX_DIR_INGRESS, 2},
67 	{MLXSW_REG_SBXX_DIR_INGRESS, 3},
68 	{MLXSW_REG_SBXX_DIR_EGRESS, 0},
69 	{MLXSW_REG_SBXX_DIR_EGRESS, 1},
70 	{MLXSW_REG_SBXX_DIR_EGRESS, 2},
71 	{MLXSW_REG_SBXX_DIR_EGRESS, 3},
72 	{MLXSW_REG_SBXX_DIR_EGRESS, 15},
73 	{MLXSW_REG_SBXX_DIR_INGRESS, 4},
74 	{MLXSW_REG_SBXX_DIR_EGRESS, 4},
75 };
76 
77 static const struct mlxsw_sp_sb_pool_des mlxsw_sp2_sb_pool_dess[] = {
78 	{MLXSW_REG_SBXX_DIR_INGRESS, 0},
79 	{MLXSW_REG_SBXX_DIR_INGRESS, 1},
80 	{MLXSW_REG_SBXX_DIR_INGRESS, 2},
81 	{MLXSW_REG_SBXX_DIR_INGRESS, 3},
82 	{MLXSW_REG_SBXX_DIR_EGRESS, 0},
83 	{MLXSW_REG_SBXX_DIR_EGRESS, 1},
84 	{MLXSW_REG_SBXX_DIR_EGRESS, 2},
85 	{MLXSW_REG_SBXX_DIR_EGRESS, 3},
86 	{MLXSW_REG_SBXX_DIR_EGRESS, 15},
87 	{MLXSW_REG_SBXX_DIR_INGRESS, 4},
88 	{MLXSW_REG_SBXX_DIR_EGRESS, 4},
89 };
90 
91 #define MLXSW_SP_SB_ING_TC_COUNT 8
92 #define MLXSW_SP_SB_EG_TC_COUNT 16
93 
94 struct mlxsw_sp_sb_port {
95 	struct mlxsw_sp_sb_cm ing_cms[MLXSW_SP_SB_ING_TC_COUNT];
96 	struct mlxsw_sp_sb_cm eg_cms[MLXSW_SP_SB_EG_TC_COUNT];
97 	struct mlxsw_sp_sb_pm *pms;
98 };
99 
100 struct mlxsw_sp_sb {
101 	struct mlxsw_sp_sb_pr *prs;
102 	struct mlxsw_sp_sb_port *ports;
103 	u32 cell_size;
104 	u32 max_headroom_cells;
105 	u64 sb_size;
106 };
107 
108 struct mlxsw_sp_sb_vals {
109 	unsigned int pool_count;
110 	const struct mlxsw_sp_sb_pool_des *pool_dess;
111 	const struct mlxsw_sp_sb_pm *pms;
112 	const struct mlxsw_sp_sb_pm *pms_cpu;
113 	const struct mlxsw_sp_sb_pr *prs;
114 	const struct mlxsw_sp_sb_mm *mms;
115 	const struct mlxsw_sp_sb_cm *cms_ingress;
116 	const struct mlxsw_sp_sb_cm *cms_egress;
117 	const struct mlxsw_sp_sb_cm *cms_cpu;
118 	unsigned int mms_count;
119 	unsigned int cms_ingress_count;
120 	unsigned int cms_egress_count;
121 	unsigned int cms_cpu_count;
122 };
123 
124 struct mlxsw_sp_sb_ops {
125 	u32 (*int_buf_size_get)(int mtu, u32 speed);
126 };
127 
128 u32 mlxsw_sp_cells_bytes(const struct mlxsw_sp *mlxsw_sp, u32 cells)
129 {
130 	return mlxsw_sp->sb->cell_size * cells;
131 }
132 
133 u32 mlxsw_sp_bytes_cells(const struct mlxsw_sp *mlxsw_sp, u32 bytes)
134 {
135 	return DIV_ROUND_UP(bytes, mlxsw_sp->sb->cell_size);
136 }
137 
138 static u32 mlxsw_sp_port_headroom_8x_adjust(const struct mlxsw_sp_port *mlxsw_sp_port,
139 					    u32 size_cells)
140 {
141 	/* Ports with eight lanes use two headroom buffers between which the
142 	 * configured headroom size is split. Therefore, multiply the calculated
143 	 * headroom size by two.
144 	 */
145 	return mlxsw_sp_port->mapping.width == 8 ? 2 * size_cells : size_cells;
146 }
147 
148 static struct mlxsw_sp_sb_pr *mlxsw_sp_sb_pr_get(struct mlxsw_sp *mlxsw_sp,
149 						 u16 pool_index)
150 {
151 	return &mlxsw_sp->sb->prs[pool_index];
152 }
153 
154 static bool mlxsw_sp_sb_cm_exists(u8 pg_buff, enum mlxsw_reg_sbxx_dir dir)
155 {
156 	if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
157 		return pg_buff < MLXSW_SP_SB_ING_TC_COUNT;
158 	else
159 		return pg_buff < MLXSW_SP_SB_EG_TC_COUNT;
160 }
161 
162 static struct mlxsw_sp_sb_cm *mlxsw_sp_sb_cm_get(struct mlxsw_sp *mlxsw_sp,
163 						 u8 local_port, u8 pg_buff,
164 						 enum mlxsw_reg_sbxx_dir dir)
165 {
166 	struct mlxsw_sp_sb_port *sb_port = &mlxsw_sp->sb->ports[local_port];
167 
168 	WARN_ON(!mlxsw_sp_sb_cm_exists(pg_buff, dir));
169 	if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
170 		return &sb_port->ing_cms[pg_buff];
171 	else
172 		return &sb_port->eg_cms[pg_buff];
173 }
174 
175 static struct mlxsw_sp_sb_pm *mlxsw_sp_sb_pm_get(struct mlxsw_sp *mlxsw_sp,
176 						 u8 local_port, u16 pool_index)
177 {
178 	return &mlxsw_sp->sb->ports[local_port].pms[pool_index];
179 }
180 
181 static int mlxsw_sp_sb_pr_write(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
182 				enum mlxsw_reg_sbpr_mode mode,
183 				u32 size, bool infi_size)
184 {
185 	const struct mlxsw_sp_sb_pool_des *des =
186 		&mlxsw_sp->sb_vals->pool_dess[pool_index];
187 	char sbpr_pl[MLXSW_REG_SBPR_LEN];
188 	struct mlxsw_sp_sb_pr *pr;
189 	int err;
190 
191 	mlxsw_reg_sbpr_pack(sbpr_pl, des->pool, des->dir, mode,
192 			    size, infi_size);
193 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbpr), sbpr_pl);
194 	if (err)
195 		return err;
196 
197 	if (infi_size)
198 		size = mlxsw_sp_bytes_cells(mlxsw_sp, mlxsw_sp->sb->sb_size);
199 	pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
200 	pr->mode = mode;
201 	pr->size = size;
202 	return 0;
203 }
204 
205 static int mlxsw_sp_sb_cm_write(struct mlxsw_sp *mlxsw_sp, u8 local_port,
206 				u8 pg_buff, u32 min_buff, u32 max_buff,
207 				bool infi_max, u16 pool_index)
208 {
209 	const struct mlxsw_sp_sb_pool_des *des =
210 		&mlxsw_sp->sb_vals->pool_dess[pool_index];
211 	char sbcm_pl[MLXSW_REG_SBCM_LEN];
212 	struct mlxsw_sp_sb_cm *cm;
213 	int err;
214 
215 	mlxsw_reg_sbcm_pack(sbcm_pl, local_port, pg_buff, des->dir,
216 			    min_buff, max_buff, infi_max, des->pool);
217 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbcm), sbcm_pl);
218 	if (err)
219 		return err;
220 
221 	if (mlxsw_sp_sb_cm_exists(pg_buff, des->dir)) {
222 		if (infi_max)
223 			max_buff = mlxsw_sp_bytes_cells(mlxsw_sp,
224 							mlxsw_sp->sb->sb_size);
225 
226 		cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, pg_buff,
227 					des->dir);
228 		cm->min_buff = min_buff;
229 		cm->max_buff = max_buff;
230 		cm->pool_index = pool_index;
231 	}
232 	return 0;
233 }
234 
235 static int mlxsw_sp_sb_pm_write(struct mlxsw_sp *mlxsw_sp, u8 local_port,
236 				u16 pool_index, u32 min_buff, u32 max_buff)
237 {
238 	const struct mlxsw_sp_sb_pool_des *des =
239 		&mlxsw_sp->sb_vals->pool_dess[pool_index];
240 	char sbpm_pl[MLXSW_REG_SBPM_LEN];
241 	struct mlxsw_sp_sb_pm *pm;
242 	int err;
243 
244 	mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir, false,
245 			    min_buff, max_buff);
246 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl);
247 	if (err)
248 		return err;
249 
250 	pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port, pool_index);
251 	pm->min_buff = min_buff;
252 	pm->max_buff = max_buff;
253 	return 0;
254 }
255 
256 static int mlxsw_sp_sb_pm_occ_clear(struct mlxsw_sp *mlxsw_sp, u8 local_port,
257 				    u16 pool_index, struct list_head *bulk_list)
258 {
259 	const struct mlxsw_sp_sb_pool_des *des =
260 		&mlxsw_sp->sb_vals->pool_dess[pool_index];
261 	char sbpm_pl[MLXSW_REG_SBPM_LEN];
262 
263 	if (local_port == MLXSW_PORT_CPU_PORT &&
264 	    des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
265 		return 0;
266 
267 	mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir,
268 			    true, 0, 0);
269 	return mlxsw_reg_trans_query(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl,
270 				     bulk_list, NULL, 0);
271 }
272 
273 static void mlxsw_sp_sb_pm_occ_query_cb(struct mlxsw_core *mlxsw_core,
274 					char *sbpm_pl, size_t sbpm_pl_len,
275 					unsigned long cb_priv)
276 {
277 	struct mlxsw_sp_sb_pm *pm = (struct mlxsw_sp_sb_pm *) cb_priv;
278 
279 	mlxsw_reg_sbpm_unpack(sbpm_pl, &pm->occ.cur, &pm->occ.max);
280 }
281 
282 static int mlxsw_sp_sb_pm_occ_query(struct mlxsw_sp *mlxsw_sp, u8 local_port,
283 				    u16 pool_index, struct list_head *bulk_list)
284 {
285 	const struct mlxsw_sp_sb_pool_des *des =
286 		&mlxsw_sp->sb_vals->pool_dess[pool_index];
287 	char sbpm_pl[MLXSW_REG_SBPM_LEN];
288 	struct mlxsw_sp_sb_pm *pm;
289 
290 	if (local_port == MLXSW_PORT_CPU_PORT &&
291 	    des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
292 		return 0;
293 
294 	pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port, pool_index);
295 	mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir,
296 			    false, 0, 0);
297 	return mlxsw_reg_trans_query(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl,
298 				     bulk_list,
299 				     mlxsw_sp_sb_pm_occ_query_cb,
300 				     (unsigned long) pm);
301 }
302 
303 void mlxsw_sp_hdroom_prios_reset_buf_idx(struct mlxsw_sp_hdroom *hdroom)
304 {
305 	int prio;
306 
307 	for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++) {
308 		switch (hdroom->mode) {
309 		case MLXSW_SP_HDROOM_MODE_DCB:
310 			hdroom->prios.prio[prio].buf_idx = hdroom->prios.prio[prio].ets_buf_idx;
311 			break;
312 		case MLXSW_SP_HDROOM_MODE_TC:
313 			hdroom->prios.prio[prio].buf_idx = hdroom->prios.prio[prio].set_buf_idx;
314 			break;
315 		}
316 	}
317 }
318 
319 void mlxsw_sp_hdroom_bufs_reset_lossiness(struct mlxsw_sp_hdroom *hdroom)
320 {
321 	int prio;
322 	int i;
323 
324 	for (i = 0; i < DCBX_MAX_BUFFERS; i++)
325 		hdroom->bufs.buf[i].lossy = true;
326 
327 	for (prio = 0; prio < IEEE_8021Q_MAX_PRIORITIES; prio++) {
328 		if (!hdroom->prios.prio[prio].lossy)
329 			hdroom->bufs.buf[hdroom->prios.prio[prio].buf_idx].lossy = false;
330 	}
331 }
332 
333 static u16 mlxsw_sp_hdroom_buf_threshold_get(const struct mlxsw_sp *mlxsw_sp, int mtu)
334 {
335 	return 2 * mlxsw_sp_bytes_cells(mlxsw_sp, mtu);
336 }
337 
338 static void mlxsw_sp_hdroom_buf_pack(char *pbmc_pl, int index, u16 size, u16 thres, bool lossy)
339 {
340 	if (lossy)
341 		mlxsw_reg_pbmc_lossy_buffer_pack(pbmc_pl, index, size);
342 	else
343 		mlxsw_reg_pbmc_lossless_buffer_pack(pbmc_pl, index, size,
344 						    thres);
345 }
346 
347 static u16 mlxsw_sp_hdroom_buf_delay_get(const struct mlxsw_sp *mlxsw_sp,
348 					 const struct mlxsw_sp_hdroom *hdroom)
349 {
350 	u16 delay_cells;
351 
352 	delay_cells = mlxsw_sp_bytes_cells(mlxsw_sp, hdroom->delay_bytes);
353 
354 	/* In the worst case scenario the delay will be made up of packets that
355 	 * are all of size CELL_SIZE + 1, which means each packet will require
356 	 * almost twice its true size when buffered in the switch. We therefore
357 	 * multiply this value by the "cell factor", which is close to 2.
358 	 *
359 	 * Another MTU is added in case the transmitting host already started
360 	 * transmitting a maximum length frame when the PFC packet was received.
361 	 */
362 	return 2 * delay_cells + mlxsw_sp_bytes_cells(mlxsw_sp, hdroom->mtu);
363 }
364 
365 static u32 mlxsw_sp_hdroom_int_buf_size_get(struct mlxsw_sp *mlxsw_sp, int mtu, u32 speed)
366 {
367 	u32 buffsize = mlxsw_sp->sb_ops->int_buf_size_get(mtu, speed);
368 
369 	return mlxsw_sp_bytes_cells(mlxsw_sp, buffsize) + 1;
370 }
371 
372 static bool mlxsw_sp_hdroom_buf_is_used(const struct mlxsw_sp_hdroom *hdroom, int buf)
373 {
374 	int prio;
375 
376 	for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++) {
377 		if (hdroom->prios.prio[prio].buf_idx == buf)
378 			return true;
379 	}
380 	return false;
381 }
382 
383 void mlxsw_sp_hdroom_bufs_reset_sizes(struct mlxsw_sp_port *mlxsw_sp_port,
384 				      struct mlxsw_sp_hdroom *hdroom)
385 {
386 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
387 	u16 reserve_cells;
388 	int i;
389 
390 	/* Internal buffer. */
391 	reserve_cells = mlxsw_sp_hdroom_int_buf_size_get(mlxsw_sp, mlxsw_sp_port->max_mtu,
392 							 mlxsw_sp_port->max_speed);
393 	reserve_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, reserve_cells);
394 	hdroom->int_buf.reserve_cells = reserve_cells;
395 
396 	if (hdroom->int_buf.enable)
397 		hdroom->int_buf.size_cells = reserve_cells;
398 	else
399 		hdroom->int_buf.size_cells = 0;
400 
401 	/* PG buffers. */
402 	for (i = 0; i < DCBX_MAX_BUFFERS; i++) {
403 		struct mlxsw_sp_hdroom_buf *buf = &hdroom->bufs.buf[i];
404 		u16 thres_cells;
405 		u16 delay_cells;
406 
407 		if (!mlxsw_sp_hdroom_buf_is_used(hdroom, i)) {
408 			thres_cells = 0;
409 			delay_cells = 0;
410 		} else if (buf->lossy) {
411 			thres_cells = mlxsw_sp_hdroom_buf_threshold_get(mlxsw_sp, hdroom->mtu);
412 			delay_cells = 0;
413 		} else {
414 			thres_cells = mlxsw_sp_hdroom_buf_threshold_get(mlxsw_sp, hdroom->mtu);
415 			delay_cells = mlxsw_sp_hdroom_buf_delay_get(mlxsw_sp, hdroom);
416 		}
417 
418 		thres_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, thres_cells);
419 		delay_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, delay_cells);
420 
421 		buf->thres_cells = thres_cells;
422 		if (hdroom->mode == MLXSW_SP_HDROOM_MODE_DCB) {
423 			buf->size_cells = thres_cells + delay_cells;
424 		} else {
425 			/* Do not allow going below the minimum size, even if
426 			 * the user requested it.
427 			 */
428 			buf->size_cells = max(buf->set_size_cells, buf->thres_cells);
429 		}
430 	}
431 }
432 
433 #define MLXSW_SP_PB_UNUSED 8
434 
435 static int mlxsw_sp_hdroom_configure_buffers(struct mlxsw_sp_port *mlxsw_sp_port,
436 					     const struct mlxsw_sp_hdroom *hdroom, bool force)
437 {
438 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
439 	char pbmc_pl[MLXSW_REG_PBMC_LEN];
440 	bool dirty;
441 	int err;
442 	int i;
443 
444 	dirty = memcmp(&mlxsw_sp_port->hdroom->bufs, &hdroom->bufs, sizeof(hdroom->bufs));
445 	if (!dirty && !force)
446 		return 0;
447 
448 	mlxsw_reg_pbmc_pack(pbmc_pl, mlxsw_sp_port->local_port, 0xffff, 0xffff / 2);
449 	for (i = 0; i < MLXSW_SP_PB_COUNT; i++) {
450 		const struct mlxsw_sp_hdroom_buf *buf = &hdroom->bufs.buf[i];
451 
452 		if (i == MLXSW_SP_PB_UNUSED)
453 			continue;
454 
455 		mlxsw_sp_hdroom_buf_pack(pbmc_pl, i, buf->size_cells, buf->thres_cells, buf->lossy);
456 	}
457 
458 	mlxsw_reg_pbmc_lossy_buffer_pack(pbmc_pl, MLXSW_REG_PBMC_PORT_SHARED_BUF_IDX, 0);
459 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pbmc), pbmc_pl);
460 	if (err)
461 		return err;
462 
463 	mlxsw_sp_port->hdroom->bufs = hdroom->bufs;
464 	return 0;
465 }
466 
467 static int mlxsw_sp_hdroom_configure_priomap(struct mlxsw_sp_port *mlxsw_sp_port,
468 					     const struct mlxsw_sp_hdroom *hdroom, bool force)
469 {
470 	char pptb_pl[MLXSW_REG_PPTB_LEN];
471 	bool dirty;
472 	int prio;
473 	int err;
474 
475 	dirty = memcmp(&mlxsw_sp_port->hdroom->prios, &hdroom->prios, sizeof(hdroom->prios));
476 	if (!dirty && !force)
477 		return 0;
478 
479 	mlxsw_reg_pptb_pack(pptb_pl, mlxsw_sp_port->local_port);
480 	for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++)
481 		mlxsw_reg_pptb_prio_to_buff_pack(pptb_pl, prio, hdroom->prios.prio[prio].buf_idx);
482 
483 	err = mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(pptb), pptb_pl);
484 	if (err)
485 		return err;
486 
487 	mlxsw_sp_port->hdroom->prios = hdroom->prios;
488 	return 0;
489 }
490 
491 static int mlxsw_sp_hdroom_configure_int_buf(struct mlxsw_sp_port *mlxsw_sp_port,
492 					     const struct mlxsw_sp_hdroom *hdroom, bool force)
493 {
494 	char sbib_pl[MLXSW_REG_SBIB_LEN];
495 	bool dirty;
496 	int err;
497 
498 	dirty = memcmp(&mlxsw_sp_port->hdroom->int_buf, &hdroom->int_buf, sizeof(hdroom->int_buf));
499 	if (!dirty && !force)
500 		return 0;
501 
502 	mlxsw_reg_sbib_pack(sbib_pl, mlxsw_sp_port->local_port, hdroom->int_buf.size_cells);
503 	err = mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(sbib), sbib_pl);
504 	if (err)
505 		return err;
506 
507 	mlxsw_sp_port->hdroom->int_buf = hdroom->int_buf;
508 	return 0;
509 }
510 
511 static bool mlxsw_sp_hdroom_bufs_fit(struct mlxsw_sp *mlxsw_sp,
512 				     const struct mlxsw_sp_hdroom *hdroom)
513 {
514 	u32 taken_headroom_cells = 0;
515 	int i;
516 
517 	for (i = 0; i < MLXSW_SP_PB_COUNT; i++)
518 		taken_headroom_cells += hdroom->bufs.buf[i].size_cells;
519 
520 	taken_headroom_cells += hdroom->int_buf.reserve_cells;
521 	return taken_headroom_cells <= mlxsw_sp->sb->max_headroom_cells;
522 }
523 
524 static int __mlxsw_sp_hdroom_configure(struct mlxsw_sp_port *mlxsw_sp_port,
525 				       const struct mlxsw_sp_hdroom *hdroom, bool force)
526 {
527 	struct mlxsw_sp_hdroom orig_hdroom;
528 	struct mlxsw_sp_hdroom tmp_hdroom;
529 	int err;
530 	int i;
531 
532 	/* Port buffers need to be configured in three steps. First, all buffers
533 	 * with non-zero size are configured. Then, prio-to-buffer map is
534 	 * updated, allowing traffic to flow to the now non-zero buffers.
535 	 * Finally, zero-sized buffers are configured, because now no traffic
536 	 * should be directed to them anymore. This way, in a non-congested
537 	 * system, no packet drops are introduced by the reconfiguration.
538 	 */
539 
540 	orig_hdroom = *mlxsw_sp_port->hdroom;
541 	tmp_hdroom = orig_hdroom;
542 	for (i = 0; i < MLXSW_SP_PB_COUNT; i++) {
543 		if (hdroom->bufs.buf[i].size_cells)
544 			tmp_hdroom.bufs.buf[i] = hdroom->bufs.buf[i];
545 	}
546 
547 	if (!mlxsw_sp_hdroom_bufs_fit(mlxsw_sp_port->mlxsw_sp, &tmp_hdroom) ||
548 	    !mlxsw_sp_hdroom_bufs_fit(mlxsw_sp_port->mlxsw_sp, hdroom))
549 		return -ENOBUFS;
550 
551 	err = mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &tmp_hdroom, force);
552 	if (err)
553 		return err;
554 
555 	err = mlxsw_sp_hdroom_configure_priomap(mlxsw_sp_port, hdroom, force);
556 	if (err)
557 		goto err_configure_priomap;
558 
559 	err = mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, hdroom, false);
560 	if (err)
561 		goto err_configure_buffers;
562 
563 	err = mlxsw_sp_hdroom_configure_int_buf(mlxsw_sp_port, hdroom, false);
564 	if (err)
565 		goto err_configure_int_buf;
566 
567 	*mlxsw_sp_port->hdroom = *hdroom;
568 	return 0;
569 
570 err_configure_int_buf:
571 	mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &tmp_hdroom, false);
572 err_configure_buffers:
573 	mlxsw_sp_hdroom_configure_priomap(mlxsw_sp_port, &tmp_hdroom, false);
574 err_configure_priomap:
575 	mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &orig_hdroom, false);
576 	return err;
577 }
578 
579 int mlxsw_sp_hdroom_configure(struct mlxsw_sp_port *mlxsw_sp_port,
580 			      const struct mlxsw_sp_hdroom *hdroom)
581 {
582 	return __mlxsw_sp_hdroom_configure(mlxsw_sp_port, hdroom, false);
583 }
584 
585 static int mlxsw_sp_port_headroom_init(struct mlxsw_sp_port *mlxsw_sp_port)
586 {
587 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
588 	struct mlxsw_sp_hdroom hdroom = {};
589 	u32 size9;
590 	int prio;
591 
592 	hdroom.mtu = mlxsw_sp_port->dev->mtu;
593 	hdroom.mode = MLXSW_SP_HDROOM_MODE_DCB;
594 	for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++)
595 		hdroom.prios.prio[prio].lossy = true;
596 
597 	mlxsw_sp_hdroom_bufs_reset_lossiness(&hdroom);
598 	mlxsw_sp_hdroom_bufs_reset_sizes(mlxsw_sp_port, &hdroom);
599 
600 	/* Buffer 9 is used for control traffic. */
601 	size9 = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, mlxsw_sp_port->max_mtu);
602 	hdroom.bufs.buf[9].size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size9);
603 
604 	return __mlxsw_sp_hdroom_configure(mlxsw_sp_port, &hdroom, true);
605 }
606 
607 static int mlxsw_sp_sb_port_init(struct mlxsw_sp *mlxsw_sp,
608 				 struct mlxsw_sp_sb_port *sb_port)
609 {
610 	struct mlxsw_sp_sb_pm *pms;
611 
612 	pms = kcalloc(mlxsw_sp->sb_vals->pool_count, sizeof(*pms),
613 		      GFP_KERNEL);
614 	if (!pms)
615 		return -ENOMEM;
616 	sb_port->pms = pms;
617 	return 0;
618 }
619 
620 static void mlxsw_sp_sb_port_fini(struct mlxsw_sp_sb_port *sb_port)
621 {
622 	kfree(sb_port->pms);
623 }
624 
625 static int mlxsw_sp_sb_ports_init(struct mlxsw_sp *mlxsw_sp)
626 {
627 	unsigned int max_ports = mlxsw_core_max_ports(mlxsw_sp->core);
628 	struct mlxsw_sp_sb_pr *prs;
629 	int i;
630 	int err;
631 
632 	mlxsw_sp->sb->ports = kcalloc(max_ports,
633 				      sizeof(struct mlxsw_sp_sb_port),
634 				      GFP_KERNEL);
635 	if (!mlxsw_sp->sb->ports)
636 		return -ENOMEM;
637 
638 	prs = kcalloc(mlxsw_sp->sb_vals->pool_count, sizeof(*prs),
639 		      GFP_KERNEL);
640 	if (!prs) {
641 		err = -ENOMEM;
642 		goto err_alloc_prs;
643 	}
644 	mlxsw_sp->sb->prs = prs;
645 
646 	for (i = 0; i < max_ports; i++) {
647 		err = mlxsw_sp_sb_port_init(mlxsw_sp, &mlxsw_sp->sb->ports[i]);
648 		if (err)
649 			goto err_sb_port_init;
650 	}
651 
652 	return 0;
653 
654 err_sb_port_init:
655 	for (i--; i >= 0; i--)
656 		mlxsw_sp_sb_port_fini(&mlxsw_sp->sb->ports[i]);
657 	kfree(mlxsw_sp->sb->prs);
658 err_alloc_prs:
659 	kfree(mlxsw_sp->sb->ports);
660 	return err;
661 }
662 
663 static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp)
664 {
665 	int max_ports = mlxsw_core_max_ports(mlxsw_sp->core);
666 	int i;
667 
668 	for (i = max_ports - 1; i >= 0; i--)
669 		mlxsw_sp_sb_port_fini(&mlxsw_sp->sb->ports[i]);
670 	kfree(mlxsw_sp->sb->prs);
671 	kfree(mlxsw_sp->sb->ports);
672 }
673 
674 #define MLXSW_SP_SB_PR(_mode, _size)	\
675 	{				\
676 		.mode = _mode,		\
677 		.size = _size,		\
678 	}
679 
680 #define MLXSW_SP_SB_PR_EXT(_mode, _size, _freeze_mode, _freeze_size)	\
681 	{								\
682 		.mode = _mode,						\
683 		.size = _size,						\
684 		.freeze_mode = _freeze_mode,				\
685 		.freeze_size = _freeze_size,				\
686 	}
687 
688 #define MLXSW_SP1_SB_PR_CPU_SIZE	(256 * 1000)
689 
690 /* Order according to mlxsw_sp1_sb_pool_dess */
691 static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = {
692 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),
693 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
694 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
695 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
696 	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,
697 			   true, false),
698 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
699 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
700 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
701 	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI,
702 			   true, true),
703 	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
704 			   MLXSW_SP1_SB_PR_CPU_SIZE, true, false),
705 	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
706 			   MLXSW_SP1_SB_PR_CPU_SIZE, true, false),
707 };
708 
709 #define MLXSW_SP2_SB_PR_CPU_SIZE	(256 * 1000)
710 
711 /* Order according to mlxsw_sp2_sb_pool_dess */
712 static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = {
713 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),
714 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
715 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
716 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
717 	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,
718 			   true, false),
719 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
720 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
721 	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
722 	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI,
723 			   true, true),
724 	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
725 			   MLXSW_SP2_SB_PR_CPU_SIZE, true, false),
726 	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
727 			   MLXSW_SP2_SB_PR_CPU_SIZE, true, false),
728 };
729 
730 static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp,
731 				const struct mlxsw_sp_sb_pr *prs,
732 				const struct mlxsw_sp_sb_pool_des *pool_dess,
733 				size_t prs_len)
734 {
735 	/* Round down, unlike mlxsw_sp_bytes_cells(). */
736 	u32 sb_cells = div_u64(mlxsw_sp->sb->sb_size, mlxsw_sp->sb->cell_size);
737 	u32 rest_cells[2] = {sb_cells, sb_cells};
738 	int i;
739 	int err;
740 
741 	/* Calculate how much space to give to the "REST" pools in either
742 	 * direction.
743 	 */
744 	for (i = 0; i < prs_len; i++) {
745 		enum mlxsw_reg_sbxx_dir dir = pool_dess[i].dir;
746 		u32 size = prs[i].size;
747 		u32 size_cells;
748 
749 		if (size == MLXSW_SP_SB_INFI || size == MLXSW_SP_SB_REST)
750 			continue;
751 
752 		size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);
753 		if (WARN_ON_ONCE(size_cells > rest_cells[dir]))
754 			continue;
755 
756 		rest_cells[dir] -= size_cells;
757 	}
758 
759 	for (i = 0; i < prs_len; i++) {
760 		u32 size = prs[i].size;
761 		u32 size_cells;
762 
763 		if (size == MLXSW_SP_SB_INFI) {
764 			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
765 						   0, true);
766 		} else if (size == MLXSW_SP_SB_REST) {
767 			size_cells = rest_cells[pool_dess[i].dir];
768 			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
769 						   size_cells, false);
770 		} else {
771 			size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);
772 			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
773 						   size_cells, false);
774 		}
775 		if (err)
776 			return err;
777 	}
778 	return 0;
779 }
780 
781 #define MLXSW_SP_SB_CM(_min_buff, _max_buff, _pool)	\
782 	{						\
783 		.min_buff = _min_buff,			\
784 		.max_buff = _max_buff,			\
785 		.pool_index = _pool,			\
786 	}
787 
788 #define MLXSW_SP_SB_CM_ING(_min_buff, _max_buff)	\
789 	{						\
790 		.min_buff = _min_buff,			\
791 		.max_buff = _max_buff,			\
792 		.pool_index = MLXSW_SP_SB_POOL_ING,	\
793 	}
794 
795 #define MLXSW_SP_SB_CM_EGR(_min_buff, _max_buff)	\
796 	{						\
797 		.min_buff = _min_buff,			\
798 		.max_buff = _max_buff,			\
799 		.pool_index = MLXSW_SP_SB_POOL_EGR,	\
800 	}
801 
802 #define MLXSW_SP_SB_CM_EGR_MC(_min_buff, _max_buff)	\
803 	{						\
804 		.min_buff = _min_buff,			\
805 		.max_buff = _max_buff,			\
806 		.pool_index = MLXSW_SP_SB_POOL_EGR_MC,	\
807 		.freeze_pool = true,			\
808 		.freeze_thresh = true,			\
809 	}
810 
811 static const struct mlxsw_sp_sb_cm mlxsw_sp1_sb_cms_ingress[] = {
812 	MLXSW_SP_SB_CM_ING(10000, 8),
813 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
814 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
815 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
816 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
817 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
818 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
819 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
820 	MLXSW_SP_SB_CM_ING(0, 0), /* dummy, this PG does not exist */
821 	MLXSW_SP_SB_CM(10000, 8, MLXSW_SP_SB_POOL_ING_CPU),
822 };
823 
824 static const struct mlxsw_sp_sb_cm mlxsw_sp2_sb_cms_ingress[] = {
825 	MLXSW_SP_SB_CM_ING(0, 7),
826 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
827 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
828 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
829 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
830 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
831 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
832 	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
833 	MLXSW_SP_SB_CM_ING(0, 0), /* dummy, this PG does not exist */
834 	MLXSW_SP_SB_CM(10000, 8, MLXSW_SP_SB_POOL_ING_CPU),
835 };
836 
837 static const struct mlxsw_sp_sb_cm mlxsw_sp1_sb_cms_egress[] = {
838 	MLXSW_SP_SB_CM_EGR(1500, 9),
839 	MLXSW_SP_SB_CM_EGR(1500, 9),
840 	MLXSW_SP_SB_CM_EGR(1500, 9),
841 	MLXSW_SP_SB_CM_EGR(1500, 9),
842 	MLXSW_SP_SB_CM_EGR(1500, 9),
843 	MLXSW_SP_SB_CM_EGR(1500, 9),
844 	MLXSW_SP_SB_CM_EGR(1500, 9),
845 	MLXSW_SP_SB_CM_EGR(1500, 9),
846 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
847 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
848 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
849 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
850 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
851 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
852 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
853 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
854 	MLXSW_SP_SB_CM_EGR(1, 0xff),
855 };
856 
857 static const struct mlxsw_sp_sb_cm mlxsw_sp2_sb_cms_egress[] = {
858 	MLXSW_SP_SB_CM_EGR(0, 7),
859 	MLXSW_SP_SB_CM_EGR(0, 7),
860 	MLXSW_SP_SB_CM_EGR(0, 7),
861 	MLXSW_SP_SB_CM_EGR(0, 7),
862 	MLXSW_SP_SB_CM_EGR(0, 7),
863 	MLXSW_SP_SB_CM_EGR(0, 7),
864 	MLXSW_SP_SB_CM_EGR(0, 7),
865 	MLXSW_SP_SB_CM_EGR(0, 7),
866 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
867 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
868 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
869 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
870 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
871 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
872 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
873 	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
874 	MLXSW_SP_SB_CM_EGR(1, 0xff),
875 };
876 
877 #define MLXSW_SP_CPU_PORT_SB_CM MLXSW_SP_SB_CM(0, 0, MLXSW_SP_SB_POOL_EGR_CPU)
878 
879 static const struct mlxsw_sp_sb_cm mlxsw_sp_cpu_port_sb_cms[] = {
880 	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
881 	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
882 	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
883 	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
884 	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
885 	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
886 	MLXSW_SP_CPU_PORT_SB_CM,
887 	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
888 	MLXSW_SP_CPU_PORT_SB_CM,
889 	MLXSW_SP_CPU_PORT_SB_CM,
890 	MLXSW_SP_CPU_PORT_SB_CM,
891 	MLXSW_SP_CPU_PORT_SB_CM,
892 	MLXSW_SP_CPU_PORT_SB_CM,
893 	MLXSW_SP_CPU_PORT_SB_CM,
894 	MLXSW_SP_CPU_PORT_SB_CM,
895 	MLXSW_SP_CPU_PORT_SB_CM,
896 	MLXSW_SP_CPU_PORT_SB_CM,
897 	MLXSW_SP_CPU_PORT_SB_CM,
898 	MLXSW_SP_CPU_PORT_SB_CM,
899 	MLXSW_SP_CPU_PORT_SB_CM,
900 	MLXSW_SP_CPU_PORT_SB_CM,
901 	MLXSW_SP_CPU_PORT_SB_CM,
902 	MLXSW_SP_CPU_PORT_SB_CM,
903 	MLXSW_SP_CPU_PORT_SB_CM,
904 	MLXSW_SP_CPU_PORT_SB_CM,
905 	MLXSW_SP_CPU_PORT_SB_CM,
906 	MLXSW_SP_CPU_PORT_SB_CM,
907 	MLXSW_SP_CPU_PORT_SB_CM,
908 	MLXSW_SP_CPU_PORT_SB_CM,
909 	MLXSW_SP_CPU_PORT_SB_CM,
910 	MLXSW_SP_CPU_PORT_SB_CM,
911 	MLXSW_SP_CPU_PORT_SB_CM,
912 };
913 
914 static bool
915 mlxsw_sp_sb_pool_is_static(struct mlxsw_sp *mlxsw_sp, u16 pool_index)
916 {
917 	struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
918 
919 	return pr->mode == MLXSW_REG_SBPR_MODE_STATIC;
920 }
921 
922 static int __mlxsw_sp_sb_cms_init(struct mlxsw_sp *mlxsw_sp, u8 local_port,
923 				  enum mlxsw_reg_sbxx_dir dir,
924 				  const struct mlxsw_sp_sb_cm *cms,
925 				  size_t cms_len)
926 {
927 	const struct mlxsw_sp_sb_vals *sb_vals = mlxsw_sp->sb_vals;
928 	int i;
929 	int err;
930 
931 	for (i = 0; i < cms_len; i++) {
932 		const struct mlxsw_sp_sb_cm *cm;
933 		u32 min_buff;
934 		u32 max_buff;
935 
936 		if (i == 8 && dir == MLXSW_REG_SBXX_DIR_INGRESS)
937 			continue; /* PG number 8 does not exist, skip it */
938 		cm = &cms[i];
939 		if (WARN_ON(sb_vals->pool_dess[cm->pool_index].dir != dir))
940 			continue;
941 
942 		min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, cm->min_buff);
943 		max_buff = cm->max_buff;
944 		if (max_buff == MLXSW_SP_SB_INFI) {
945 			err = mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, i,
946 						   min_buff, 0,
947 						   true, cm->pool_index);
948 		} else {
949 			if (mlxsw_sp_sb_pool_is_static(mlxsw_sp,
950 						       cm->pool_index))
951 				max_buff = mlxsw_sp_bytes_cells(mlxsw_sp,
952 								max_buff);
953 			err = mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, i,
954 						   min_buff, max_buff,
955 						   false, cm->pool_index);
956 		}
957 		if (err)
958 			return err;
959 	}
960 	return 0;
961 }
962 
963 static int mlxsw_sp_port_sb_cms_init(struct mlxsw_sp_port *mlxsw_sp_port)
964 {
965 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
966 	int err;
967 
968 	err = __mlxsw_sp_sb_cms_init(mlxsw_sp,
969 				     mlxsw_sp_port->local_port,
970 				     MLXSW_REG_SBXX_DIR_INGRESS,
971 				     mlxsw_sp->sb_vals->cms_ingress,
972 				     mlxsw_sp->sb_vals->cms_ingress_count);
973 	if (err)
974 		return err;
975 	return __mlxsw_sp_sb_cms_init(mlxsw_sp_port->mlxsw_sp,
976 				      mlxsw_sp_port->local_port,
977 				      MLXSW_REG_SBXX_DIR_EGRESS,
978 				      mlxsw_sp->sb_vals->cms_egress,
979 				      mlxsw_sp->sb_vals->cms_egress_count);
980 }
981 
982 static int mlxsw_sp_cpu_port_sb_cms_init(struct mlxsw_sp *mlxsw_sp)
983 {
984 	return __mlxsw_sp_sb_cms_init(mlxsw_sp, 0, MLXSW_REG_SBXX_DIR_EGRESS,
985 				      mlxsw_sp->sb_vals->cms_cpu,
986 				      mlxsw_sp->sb_vals->cms_cpu_count);
987 }
988 
989 #define MLXSW_SP_SB_PM(_min_buff, _max_buff)	\
990 	{					\
991 		.min_buff = _min_buff,		\
992 		.max_buff = _max_buff,		\
993 	}
994 
995 /* Order according to mlxsw_sp1_sb_pool_dess */
996 static const struct mlxsw_sp_sb_pm mlxsw_sp1_sb_pms[] = {
997 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX),
998 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
999 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1000 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1001 	MLXSW_SP_SB_PM(0, 7),
1002 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1003 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1004 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1005 	MLXSW_SP_SB_PM(10000, 90000),
1006 	MLXSW_SP_SB_PM(0, 8),	/* 50% occupancy */
1007 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1008 };
1009 
1010 /* Order according to mlxsw_sp2_sb_pool_dess */
1011 static const struct mlxsw_sp_sb_pm mlxsw_sp2_sb_pms[] = {
1012 	MLXSW_SP_SB_PM(0, 7),
1013 	MLXSW_SP_SB_PM(0, 0),
1014 	MLXSW_SP_SB_PM(0, 0),
1015 	MLXSW_SP_SB_PM(0, 0),
1016 	MLXSW_SP_SB_PM(0, 7),
1017 	MLXSW_SP_SB_PM(0, 0),
1018 	MLXSW_SP_SB_PM(0, 0),
1019 	MLXSW_SP_SB_PM(0, 0),
1020 	MLXSW_SP_SB_PM(10000, 90000),
1021 	MLXSW_SP_SB_PM(0, 8),	/* 50% occupancy */
1022 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1023 };
1024 
1025 /* Order according to mlxsw_sp*_sb_pool_dess */
1026 static const struct mlxsw_sp_sb_pm mlxsw_sp_cpu_port_sb_pms[] = {
1027 	MLXSW_SP_SB_PM(0, 0),
1028 	MLXSW_SP_SB_PM(0, 0),
1029 	MLXSW_SP_SB_PM(0, 0),
1030 	MLXSW_SP_SB_PM(0, 0),
1031 	MLXSW_SP_SB_PM(0, 0),
1032 	MLXSW_SP_SB_PM(0, 0),
1033 	MLXSW_SP_SB_PM(0, 0),
1034 	MLXSW_SP_SB_PM(0, 0),
1035 	MLXSW_SP_SB_PM(0, 90000),
1036 	MLXSW_SP_SB_PM(0, 0),
1037 	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX),
1038 };
1039 
1040 static int mlxsw_sp_sb_pms_init(struct mlxsw_sp *mlxsw_sp, u8 local_port,
1041 				const struct mlxsw_sp_sb_pm *pms,
1042 				bool skip_ingress)
1043 {
1044 	int i, err;
1045 
1046 	for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1047 		const struct mlxsw_sp_sb_pm *pm = &pms[i];
1048 		const struct mlxsw_sp_sb_pool_des *des;
1049 		u32 max_buff;
1050 		u32 min_buff;
1051 
1052 		des = &mlxsw_sp->sb_vals->pool_dess[i];
1053 		if (skip_ingress && des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
1054 			continue;
1055 
1056 		min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, pm->min_buff);
1057 		max_buff = pm->max_buff;
1058 		if (mlxsw_sp_sb_pool_is_static(mlxsw_sp, i))
1059 			max_buff = mlxsw_sp_bytes_cells(mlxsw_sp, max_buff);
1060 		err = mlxsw_sp_sb_pm_write(mlxsw_sp, local_port, i, min_buff,
1061 					   max_buff);
1062 		if (err)
1063 			return err;
1064 	}
1065 	return 0;
1066 }
1067 
1068 static int mlxsw_sp_port_sb_pms_init(struct mlxsw_sp_port *mlxsw_sp_port)
1069 {
1070 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1071 
1072 	return mlxsw_sp_sb_pms_init(mlxsw_sp, mlxsw_sp_port->local_port,
1073 				    mlxsw_sp->sb_vals->pms, false);
1074 }
1075 
1076 static int mlxsw_sp_cpu_port_sb_pms_init(struct mlxsw_sp *mlxsw_sp)
1077 {
1078 	return mlxsw_sp_sb_pms_init(mlxsw_sp, 0, mlxsw_sp->sb_vals->pms_cpu,
1079 				    true);
1080 }
1081 
1082 #define MLXSW_SP_SB_MM(_min_buff, _max_buff)		\
1083 	{						\
1084 		.min_buff = _min_buff,			\
1085 		.max_buff = _max_buff,			\
1086 		.pool_index = MLXSW_SP_SB_POOL_EGR,	\
1087 	}
1088 
1089 static const struct mlxsw_sp_sb_mm mlxsw_sp_sb_mms[] = {
1090 	MLXSW_SP_SB_MM(0, 6),
1091 	MLXSW_SP_SB_MM(0, 6),
1092 	MLXSW_SP_SB_MM(0, 6),
1093 	MLXSW_SP_SB_MM(0, 6),
1094 	MLXSW_SP_SB_MM(0, 6),
1095 	MLXSW_SP_SB_MM(0, 6),
1096 	MLXSW_SP_SB_MM(0, 6),
1097 	MLXSW_SP_SB_MM(0, 6),
1098 	MLXSW_SP_SB_MM(0, 6),
1099 	MLXSW_SP_SB_MM(0, 6),
1100 	MLXSW_SP_SB_MM(0, 6),
1101 	MLXSW_SP_SB_MM(0, 6),
1102 	MLXSW_SP_SB_MM(0, 6),
1103 	MLXSW_SP_SB_MM(0, 6),
1104 	MLXSW_SP_SB_MM(0, 6),
1105 };
1106 
1107 static int mlxsw_sp_sb_mms_init(struct mlxsw_sp *mlxsw_sp)
1108 {
1109 	char sbmm_pl[MLXSW_REG_SBMM_LEN];
1110 	int i;
1111 	int err;
1112 
1113 	for (i = 0; i < mlxsw_sp->sb_vals->mms_count; i++) {
1114 		const struct mlxsw_sp_sb_pool_des *des;
1115 		const struct mlxsw_sp_sb_mm *mc;
1116 		u32 min_buff;
1117 
1118 		mc = &mlxsw_sp->sb_vals->mms[i];
1119 		des = &mlxsw_sp->sb_vals->pool_dess[mc->pool_index];
1120 		/* All pools used by sb_mm's are initialized using dynamic
1121 		 * thresholds, therefore 'max_buff' isn't specified in cells.
1122 		 */
1123 		min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, mc->min_buff);
1124 		mlxsw_reg_sbmm_pack(sbmm_pl, i, min_buff, mc->max_buff,
1125 				    des->pool);
1126 		err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbmm), sbmm_pl);
1127 		if (err)
1128 			return err;
1129 	}
1130 	return 0;
1131 }
1132 
1133 static void mlxsw_sp_pool_count(struct mlxsw_sp *mlxsw_sp,
1134 				u16 *p_ingress_len, u16 *p_egress_len)
1135 {
1136 	int i;
1137 
1138 	for (i = 0; i < mlxsw_sp->sb_vals->pool_count; ++i) {
1139 		if (mlxsw_sp->sb_vals->pool_dess[i].dir ==
1140 		    MLXSW_REG_SBXX_DIR_INGRESS)
1141 			(*p_ingress_len)++;
1142 		else
1143 			(*p_egress_len)++;
1144 	}
1145 
1146 	WARN(*p_egress_len == 0, "No egress pools\n");
1147 }
1148 
1149 const struct mlxsw_sp_sb_vals mlxsw_sp1_sb_vals = {
1150 	.pool_count = ARRAY_SIZE(mlxsw_sp1_sb_pool_dess),
1151 	.pool_dess = mlxsw_sp1_sb_pool_dess,
1152 	.pms = mlxsw_sp1_sb_pms,
1153 	.pms_cpu = mlxsw_sp_cpu_port_sb_pms,
1154 	.prs = mlxsw_sp1_sb_prs,
1155 	.mms = mlxsw_sp_sb_mms,
1156 	.cms_ingress = mlxsw_sp1_sb_cms_ingress,
1157 	.cms_egress = mlxsw_sp1_sb_cms_egress,
1158 	.cms_cpu = mlxsw_sp_cpu_port_sb_cms,
1159 	.mms_count = ARRAY_SIZE(mlxsw_sp_sb_mms),
1160 	.cms_ingress_count = ARRAY_SIZE(mlxsw_sp1_sb_cms_ingress),
1161 	.cms_egress_count = ARRAY_SIZE(mlxsw_sp1_sb_cms_egress),
1162 	.cms_cpu_count = ARRAY_SIZE(mlxsw_sp_cpu_port_sb_cms),
1163 };
1164 
1165 const struct mlxsw_sp_sb_vals mlxsw_sp2_sb_vals = {
1166 	.pool_count = ARRAY_SIZE(mlxsw_sp2_sb_pool_dess),
1167 	.pool_dess = mlxsw_sp2_sb_pool_dess,
1168 	.pms = mlxsw_sp2_sb_pms,
1169 	.pms_cpu = mlxsw_sp_cpu_port_sb_pms,
1170 	.prs = mlxsw_sp2_sb_prs,
1171 	.mms = mlxsw_sp_sb_mms,
1172 	.cms_ingress = mlxsw_sp2_sb_cms_ingress,
1173 	.cms_egress = mlxsw_sp2_sb_cms_egress,
1174 	.cms_cpu = mlxsw_sp_cpu_port_sb_cms,
1175 	.mms_count = ARRAY_SIZE(mlxsw_sp_sb_mms),
1176 	.cms_ingress_count = ARRAY_SIZE(mlxsw_sp2_sb_cms_ingress),
1177 	.cms_egress_count = ARRAY_SIZE(mlxsw_sp2_sb_cms_egress),
1178 	.cms_cpu_count = ARRAY_SIZE(mlxsw_sp_cpu_port_sb_cms),
1179 };
1180 
1181 static u32 mlxsw_sp1_pb_int_buf_size_get(int mtu, u32 speed)
1182 {
1183 	return mtu * 5 / 2;
1184 }
1185 
1186 static u32 __mlxsw_sp_pb_int_buf_size_get(int mtu, u32 speed, u32 buffer_factor)
1187 {
1188 	return 3 * mtu + buffer_factor * speed / 1000;
1189 }
1190 
1191 #define MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR 38
1192 
1193 static u32 mlxsw_sp2_pb_int_buf_size_get(int mtu, u32 speed)
1194 {
1195 	int factor = MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR;
1196 
1197 	return __mlxsw_sp_pb_int_buf_size_get(mtu, speed, factor);
1198 }
1199 
1200 #define MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR 50
1201 
1202 static u32 mlxsw_sp3_pb_int_buf_size_get(int mtu, u32 speed)
1203 {
1204 	int factor = MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR;
1205 
1206 	return __mlxsw_sp_pb_int_buf_size_get(mtu, speed, factor);
1207 }
1208 
1209 const struct mlxsw_sp_sb_ops mlxsw_sp1_sb_ops = {
1210 	.int_buf_size_get = mlxsw_sp1_pb_int_buf_size_get,
1211 };
1212 
1213 const struct mlxsw_sp_sb_ops mlxsw_sp2_sb_ops = {
1214 	.int_buf_size_get = mlxsw_sp2_pb_int_buf_size_get,
1215 };
1216 
1217 const struct mlxsw_sp_sb_ops mlxsw_sp3_sb_ops = {
1218 	.int_buf_size_get = mlxsw_sp3_pb_int_buf_size_get,
1219 };
1220 
1221 int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp)
1222 {
1223 	u32 max_headroom_size;
1224 	u16 ing_pool_count = 0;
1225 	u16 eg_pool_count = 0;
1226 	int err;
1227 
1228 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, CELL_SIZE))
1229 		return -EIO;
1230 
1231 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, GUARANTEED_SHARED_BUFFER))
1232 		return -EIO;
1233 
1234 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_HEADROOM_SIZE))
1235 		return -EIO;
1236 
1237 	mlxsw_sp->sb = kzalloc(sizeof(*mlxsw_sp->sb), GFP_KERNEL);
1238 	if (!mlxsw_sp->sb)
1239 		return -ENOMEM;
1240 	mlxsw_sp->sb->cell_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, CELL_SIZE);
1241 	mlxsw_sp->sb->sb_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1242 						   GUARANTEED_SHARED_BUFFER);
1243 	max_headroom_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1244 					       MAX_HEADROOM_SIZE);
1245 	/* Round down, because this limit must not be overstepped. */
1246 	mlxsw_sp->sb->max_headroom_cells = max_headroom_size /
1247 						mlxsw_sp->sb->cell_size;
1248 
1249 	err = mlxsw_sp_sb_ports_init(mlxsw_sp);
1250 	if (err)
1251 		goto err_sb_ports_init;
1252 	err = mlxsw_sp_sb_prs_init(mlxsw_sp, mlxsw_sp->sb_vals->prs,
1253 				   mlxsw_sp->sb_vals->pool_dess,
1254 				   mlxsw_sp->sb_vals->pool_count);
1255 	if (err)
1256 		goto err_sb_prs_init;
1257 	err = mlxsw_sp_cpu_port_sb_cms_init(mlxsw_sp);
1258 	if (err)
1259 		goto err_sb_cpu_port_sb_cms_init;
1260 	err = mlxsw_sp_cpu_port_sb_pms_init(mlxsw_sp);
1261 	if (err)
1262 		goto err_sb_cpu_port_pms_init;
1263 	err = mlxsw_sp_sb_mms_init(mlxsw_sp);
1264 	if (err)
1265 		goto err_sb_mms_init;
1266 	mlxsw_sp_pool_count(mlxsw_sp, &ing_pool_count, &eg_pool_count);
1267 	err = devlink_sb_register(priv_to_devlink(mlxsw_sp->core), 0,
1268 				  mlxsw_sp->sb->sb_size,
1269 				  ing_pool_count,
1270 				  eg_pool_count,
1271 				  MLXSW_SP_SB_ING_TC_COUNT,
1272 				  MLXSW_SP_SB_EG_TC_COUNT);
1273 	if (err)
1274 		goto err_devlink_sb_register;
1275 
1276 	return 0;
1277 
1278 err_devlink_sb_register:
1279 err_sb_mms_init:
1280 err_sb_cpu_port_pms_init:
1281 err_sb_cpu_port_sb_cms_init:
1282 err_sb_prs_init:
1283 	mlxsw_sp_sb_ports_fini(mlxsw_sp);
1284 err_sb_ports_init:
1285 	kfree(mlxsw_sp->sb);
1286 	return err;
1287 }
1288 
1289 void mlxsw_sp_buffers_fini(struct mlxsw_sp *mlxsw_sp)
1290 {
1291 	devlink_sb_unregister(priv_to_devlink(mlxsw_sp->core), 0);
1292 	mlxsw_sp_sb_ports_fini(mlxsw_sp);
1293 	kfree(mlxsw_sp->sb);
1294 }
1295 
1296 int mlxsw_sp_port_buffers_init(struct mlxsw_sp_port *mlxsw_sp_port)
1297 {
1298 	int err;
1299 
1300 	mlxsw_sp_port->hdroom = kzalloc(sizeof(*mlxsw_sp_port->hdroom), GFP_KERNEL);
1301 	if (!mlxsw_sp_port->hdroom)
1302 		return -ENOMEM;
1303 	mlxsw_sp_port->hdroom->mtu = mlxsw_sp_port->dev->mtu;
1304 
1305 	err = mlxsw_sp_port_headroom_init(mlxsw_sp_port);
1306 	if (err)
1307 		goto err_headroom_init;
1308 	err = mlxsw_sp_port_sb_cms_init(mlxsw_sp_port);
1309 	if (err)
1310 		goto err_port_sb_cms_init;
1311 	err = mlxsw_sp_port_sb_pms_init(mlxsw_sp_port);
1312 	if (err)
1313 		goto err_port_sb_pms_init;
1314 	return 0;
1315 
1316 err_port_sb_pms_init:
1317 err_port_sb_cms_init:
1318 err_headroom_init:
1319 	kfree(mlxsw_sp_port->hdroom);
1320 	return err;
1321 }
1322 
1323 void mlxsw_sp_port_buffers_fini(struct mlxsw_sp_port *mlxsw_sp_port)
1324 {
1325 	kfree(mlxsw_sp_port->hdroom);
1326 }
1327 
1328 int mlxsw_sp_sb_pool_get(struct mlxsw_core *mlxsw_core,
1329 			 unsigned int sb_index, u16 pool_index,
1330 			 struct devlink_sb_pool_info *pool_info)
1331 {
1332 	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1333 	enum mlxsw_reg_sbxx_dir dir;
1334 	struct mlxsw_sp_sb_pr *pr;
1335 
1336 	dir = mlxsw_sp->sb_vals->pool_dess[pool_index].dir;
1337 	pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1338 	pool_info->pool_type = (enum devlink_sb_pool_type) dir;
1339 	pool_info->size = mlxsw_sp_cells_bytes(mlxsw_sp, pr->size);
1340 	pool_info->threshold_type = (enum devlink_sb_threshold_type) pr->mode;
1341 	pool_info->cell_size = mlxsw_sp->sb->cell_size;
1342 	return 0;
1343 }
1344 
1345 int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core,
1346 			 unsigned int sb_index, u16 pool_index, u32 size,
1347 			 enum devlink_sb_threshold_type threshold_type,
1348 			 struct netlink_ext_ack *extack)
1349 {
1350 	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1351 	u32 pool_size = mlxsw_sp_bytes_cells(mlxsw_sp, size);
1352 	const struct mlxsw_sp_sb_pr *pr;
1353 	enum mlxsw_reg_sbpr_mode mode;
1354 
1355 	mode = (enum mlxsw_reg_sbpr_mode) threshold_type;
1356 	pr = &mlxsw_sp->sb_vals->prs[pool_index];
1357 
1358 	if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core,
1359 				      GUARANTEED_SHARED_BUFFER)) {
1360 		NL_SET_ERR_MSG_MOD(extack, "Exceeded shared buffer size");
1361 		return -EINVAL;
1362 	}
1363 
1364 	if (pr->freeze_mode && pr->mode != mode) {
1365 		NL_SET_ERR_MSG_MOD(extack, "Changing this pool's threshold type is forbidden");
1366 		return -EINVAL;
1367 	}
1368 
1369 	if (pr->freeze_size && pr->size != size) {
1370 		NL_SET_ERR_MSG_MOD(extack, "Changing this pool's size is forbidden");
1371 		return -EINVAL;
1372 	}
1373 
1374 	return mlxsw_sp_sb_pr_write(mlxsw_sp, pool_index, mode,
1375 				    pool_size, false);
1376 }
1377 
1378 #define MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET (-2) /* 3->1, 16->14 */
1379 
1380 static u32 mlxsw_sp_sb_threshold_out(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
1381 				     u32 max_buff)
1382 {
1383 	struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1384 
1385 	if (pr->mode == MLXSW_REG_SBPR_MODE_DYNAMIC)
1386 		return max_buff - MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET;
1387 	return mlxsw_sp_cells_bytes(mlxsw_sp, max_buff);
1388 }
1389 
1390 static int mlxsw_sp_sb_threshold_in(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
1391 				    u32 threshold, u32 *p_max_buff,
1392 				    struct netlink_ext_ack *extack)
1393 {
1394 	struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1395 
1396 	if (pr->mode == MLXSW_REG_SBPR_MODE_DYNAMIC) {
1397 		int val;
1398 
1399 		val = threshold + MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET;
1400 		if (val < MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN ||
1401 		    val > MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX) {
1402 			NL_SET_ERR_MSG_MOD(extack, "Invalid dynamic threshold value");
1403 			return -EINVAL;
1404 		}
1405 		*p_max_buff = val;
1406 	} else {
1407 		*p_max_buff = mlxsw_sp_bytes_cells(mlxsw_sp, threshold);
1408 	}
1409 	return 0;
1410 }
1411 
1412 int mlxsw_sp_sb_port_pool_get(struct mlxsw_core_port *mlxsw_core_port,
1413 			      unsigned int sb_index, u16 pool_index,
1414 			      u32 *p_threshold)
1415 {
1416 	struct mlxsw_sp_port *mlxsw_sp_port =
1417 			mlxsw_core_port_driver_priv(mlxsw_core_port);
1418 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1419 	u8 local_port = mlxsw_sp_port->local_port;
1420 	struct mlxsw_sp_sb_pm *pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port,
1421 						       pool_index);
1422 
1423 	*p_threshold = mlxsw_sp_sb_threshold_out(mlxsw_sp, pool_index,
1424 						 pm->max_buff);
1425 	return 0;
1426 }
1427 
1428 int mlxsw_sp_sb_port_pool_set(struct mlxsw_core_port *mlxsw_core_port,
1429 			      unsigned int sb_index, u16 pool_index,
1430 			      u32 threshold, struct netlink_ext_ack *extack)
1431 {
1432 	struct mlxsw_sp_port *mlxsw_sp_port =
1433 			mlxsw_core_port_driver_priv(mlxsw_core_port);
1434 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1435 	u8 local_port = mlxsw_sp_port->local_port;
1436 	u32 max_buff;
1437 	int err;
1438 
1439 	if (local_port == MLXSW_PORT_CPU_PORT) {
1440 		NL_SET_ERR_MSG_MOD(extack, "Changing CPU port's threshold is forbidden");
1441 		return -EINVAL;
1442 	}
1443 
1444 	err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool_index,
1445 				       threshold, &max_buff, extack);
1446 	if (err)
1447 		return err;
1448 
1449 	return mlxsw_sp_sb_pm_write(mlxsw_sp, local_port, pool_index,
1450 				    0, max_buff);
1451 }
1452 
1453 int mlxsw_sp_sb_tc_pool_bind_get(struct mlxsw_core_port *mlxsw_core_port,
1454 				 unsigned int sb_index, u16 tc_index,
1455 				 enum devlink_sb_pool_type pool_type,
1456 				 u16 *p_pool_index, u32 *p_threshold)
1457 {
1458 	struct mlxsw_sp_port *mlxsw_sp_port =
1459 			mlxsw_core_port_driver_priv(mlxsw_core_port);
1460 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1461 	u8 local_port = mlxsw_sp_port->local_port;
1462 	u8 pg_buff = tc_index;
1463 	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1464 	struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port,
1465 						       pg_buff, dir);
1466 
1467 	*p_threshold = mlxsw_sp_sb_threshold_out(mlxsw_sp, cm->pool_index,
1468 						 cm->max_buff);
1469 	*p_pool_index = cm->pool_index;
1470 	return 0;
1471 }
1472 
1473 int mlxsw_sp_sb_tc_pool_bind_set(struct mlxsw_core_port *mlxsw_core_port,
1474 				 unsigned int sb_index, u16 tc_index,
1475 				 enum devlink_sb_pool_type pool_type,
1476 				 u16 pool_index, u32 threshold,
1477 				 struct netlink_ext_ack *extack)
1478 {
1479 	struct mlxsw_sp_port *mlxsw_sp_port =
1480 			mlxsw_core_port_driver_priv(mlxsw_core_port);
1481 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1482 	u8 local_port = mlxsw_sp_port->local_port;
1483 	const struct mlxsw_sp_sb_cm *cm;
1484 	u8 pg_buff = tc_index;
1485 	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1486 	u32 max_buff;
1487 	int err;
1488 
1489 	if (local_port == MLXSW_PORT_CPU_PORT) {
1490 		NL_SET_ERR_MSG_MOD(extack, "Changing CPU port's binding is forbidden");
1491 		return -EINVAL;
1492 	}
1493 
1494 	if (dir != mlxsw_sp->sb_vals->pool_dess[pool_index].dir) {
1495 		NL_SET_ERR_MSG_MOD(extack, "Binding egress TC to ingress pool and vice versa is forbidden");
1496 		return -EINVAL;
1497 	}
1498 
1499 	if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
1500 		cm = &mlxsw_sp->sb_vals->cms_ingress[tc_index];
1501 	else
1502 		cm = &mlxsw_sp->sb_vals->cms_egress[tc_index];
1503 
1504 	if (cm->freeze_pool && cm->pool_index != pool_index) {
1505 		NL_SET_ERR_MSG_MOD(extack, "Binding this TC to a different pool is forbidden");
1506 		return -EINVAL;
1507 	}
1508 
1509 	if (cm->freeze_thresh && cm->max_buff != threshold) {
1510 		NL_SET_ERR_MSG_MOD(extack, "Changing this TC's threshold is forbidden");
1511 		return -EINVAL;
1512 	}
1513 
1514 	err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool_index,
1515 				       threshold, &max_buff, extack);
1516 	if (err)
1517 		return err;
1518 
1519 	return mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, pg_buff,
1520 				    0, max_buff, false, pool_index);
1521 }
1522 
1523 #define MASKED_COUNT_MAX \
1524 	(MLXSW_REG_SBSR_REC_MAX_COUNT / \
1525 	 (MLXSW_SP_SB_ING_TC_COUNT + MLXSW_SP_SB_EG_TC_COUNT))
1526 
1527 struct mlxsw_sp_sb_sr_occ_query_cb_ctx {
1528 	u8 masked_count;
1529 	u8 local_port_1;
1530 };
1531 
1532 static void mlxsw_sp_sb_sr_occ_query_cb(struct mlxsw_core *mlxsw_core,
1533 					char *sbsr_pl, size_t sbsr_pl_len,
1534 					unsigned long cb_priv)
1535 {
1536 	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1537 	struct mlxsw_sp_sb_sr_occ_query_cb_ctx cb_ctx;
1538 	u8 masked_count;
1539 	u8 local_port;
1540 	int rec_index = 0;
1541 	struct mlxsw_sp_sb_cm *cm;
1542 	int i;
1543 
1544 	memcpy(&cb_ctx, &cb_priv, sizeof(cb_ctx));
1545 
1546 	masked_count = 0;
1547 	for (local_port = cb_ctx.local_port_1;
1548 	     local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1549 		if (!mlxsw_sp->ports[local_port])
1550 			continue;
1551 		if (local_port == MLXSW_PORT_CPU_PORT) {
1552 			/* Ingress quotas are not supported for the CPU port */
1553 			masked_count++;
1554 			continue;
1555 		}
1556 		for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++) {
1557 			cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, i,
1558 						MLXSW_REG_SBXX_DIR_INGRESS);
1559 			mlxsw_reg_sbsr_rec_unpack(sbsr_pl, rec_index++,
1560 						  &cm->occ.cur, &cm->occ.max);
1561 		}
1562 		if (++masked_count == cb_ctx.masked_count)
1563 			break;
1564 	}
1565 	masked_count = 0;
1566 	for (local_port = cb_ctx.local_port_1;
1567 	     local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1568 		if (!mlxsw_sp->ports[local_port])
1569 			continue;
1570 		for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++) {
1571 			cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, i,
1572 						MLXSW_REG_SBXX_DIR_EGRESS);
1573 			mlxsw_reg_sbsr_rec_unpack(sbsr_pl, rec_index++,
1574 						  &cm->occ.cur, &cm->occ.max);
1575 		}
1576 		if (++masked_count == cb_ctx.masked_count)
1577 			break;
1578 	}
1579 }
1580 
1581 int mlxsw_sp_sb_occ_snapshot(struct mlxsw_core *mlxsw_core,
1582 			     unsigned int sb_index)
1583 {
1584 	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1585 	struct mlxsw_sp_sb_sr_occ_query_cb_ctx cb_ctx;
1586 	unsigned long cb_priv = 0;
1587 	LIST_HEAD(bulk_list);
1588 	char *sbsr_pl;
1589 	u8 masked_count;
1590 	u8 local_port_1;
1591 	u8 local_port;
1592 	int i;
1593 	int err;
1594 	int err2;
1595 
1596 	sbsr_pl = kmalloc(MLXSW_REG_SBSR_LEN, GFP_KERNEL);
1597 	if (!sbsr_pl)
1598 		return -ENOMEM;
1599 
1600 	local_port = MLXSW_PORT_CPU_PORT;
1601 next_batch:
1602 	local_port_1 = local_port;
1603 	masked_count = 0;
1604 	mlxsw_reg_sbsr_pack(sbsr_pl, false);
1605 	for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++)
1606 		mlxsw_reg_sbsr_pg_buff_mask_set(sbsr_pl, i, 1);
1607 	for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++)
1608 		mlxsw_reg_sbsr_tclass_mask_set(sbsr_pl, i, 1);
1609 	for (; local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1610 		if (!mlxsw_sp->ports[local_port])
1611 			continue;
1612 		if (local_port != MLXSW_PORT_CPU_PORT) {
1613 			/* Ingress quotas are not supported for the CPU port */
1614 			mlxsw_reg_sbsr_ingress_port_mask_set(sbsr_pl,
1615 							     local_port, 1);
1616 		}
1617 		mlxsw_reg_sbsr_egress_port_mask_set(sbsr_pl, local_port, 1);
1618 		for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1619 			err = mlxsw_sp_sb_pm_occ_query(mlxsw_sp, local_port, i,
1620 						       &bulk_list);
1621 			if (err)
1622 				goto out;
1623 		}
1624 		if (++masked_count == MASKED_COUNT_MAX)
1625 			goto do_query;
1626 	}
1627 
1628 do_query:
1629 	cb_ctx.masked_count = masked_count;
1630 	cb_ctx.local_port_1 = local_port_1;
1631 	memcpy(&cb_priv, &cb_ctx, sizeof(cb_ctx));
1632 	err = mlxsw_reg_trans_query(mlxsw_core, MLXSW_REG(sbsr), sbsr_pl,
1633 				    &bulk_list, mlxsw_sp_sb_sr_occ_query_cb,
1634 				    cb_priv);
1635 	if (err)
1636 		goto out;
1637 	if (local_port < mlxsw_core_max_ports(mlxsw_core)) {
1638 		local_port++;
1639 		goto next_batch;
1640 	}
1641 
1642 out:
1643 	err2 = mlxsw_reg_trans_bulk_wait(&bulk_list);
1644 	if (!err)
1645 		err = err2;
1646 	kfree(sbsr_pl);
1647 	return err;
1648 }
1649 
1650 int mlxsw_sp_sb_occ_max_clear(struct mlxsw_core *mlxsw_core,
1651 			      unsigned int sb_index)
1652 {
1653 	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1654 	LIST_HEAD(bulk_list);
1655 	char *sbsr_pl;
1656 	unsigned int masked_count;
1657 	u8 local_port;
1658 	int i;
1659 	int err;
1660 	int err2;
1661 
1662 	sbsr_pl = kmalloc(MLXSW_REG_SBSR_LEN, GFP_KERNEL);
1663 	if (!sbsr_pl)
1664 		return -ENOMEM;
1665 
1666 	local_port = MLXSW_PORT_CPU_PORT;
1667 next_batch:
1668 	masked_count = 0;
1669 	mlxsw_reg_sbsr_pack(sbsr_pl, true);
1670 	for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++)
1671 		mlxsw_reg_sbsr_pg_buff_mask_set(sbsr_pl, i, 1);
1672 	for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++)
1673 		mlxsw_reg_sbsr_tclass_mask_set(sbsr_pl, i, 1);
1674 	for (; local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1675 		if (!mlxsw_sp->ports[local_port])
1676 			continue;
1677 		if (local_port != MLXSW_PORT_CPU_PORT) {
1678 			/* Ingress quotas are not supported for the CPU port */
1679 			mlxsw_reg_sbsr_ingress_port_mask_set(sbsr_pl,
1680 							     local_port, 1);
1681 		}
1682 		mlxsw_reg_sbsr_egress_port_mask_set(sbsr_pl, local_port, 1);
1683 		for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1684 			err = mlxsw_sp_sb_pm_occ_clear(mlxsw_sp, local_port, i,
1685 						       &bulk_list);
1686 			if (err)
1687 				goto out;
1688 		}
1689 		if (++masked_count == MASKED_COUNT_MAX)
1690 			goto do_query;
1691 	}
1692 
1693 do_query:
1694 	err = mlxsw_reg_trans_query(mlxsw_core, MLXSW_REG(sbsr), sbsr_pl,
1695 				    &bulk_list, NULL, 0);
1696 	if (err)
1697 		goto out;
1698 	if (local_port < mlxsw_core_max_ports(mlxsw_core)) {
1699 		local_port++;
1700 		goto next_batch;
1701 	}
1702 
1703 out:
1704 	err2 = mlxsw_reg_trans_bulk_wait(&bulk_list);
1705 	if (!err)
1706 		err = err2;
1707 	kfree(sbsr_pl);
1708 	return err;
1709 }
1710 
1711 int mlxsw_sp_sb_occ_port_pool_get(struct mlxsw_core_port *mlxsw_core_port,
1712 				  unsigned int sb_index, u16 pool_index,
1713 				  u32 *p_cur, u32 *p_max)
1714 {
1715 	struct mlxsw_sp_port *mlxsw_sp_port =
1716 			mlxsw_core_port_driver_priv(mlxsw_core_port);
1717 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1718 	u8 local_port = mlxsw_sp_port->local_port;
1719 	struct mlxsw_sp_sb_pm *pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port,
1720 						       pool_index);
1721 
1722 	*p_cur = mlxsw_sp_cells_bytes(mlxsw_sp, pm->occ.cur);
1723 	*p_max = mlxsw_sp_cells_bytes(mlxsw_sp, pm->occ.max);
1724 	return 0;
1725 }
1726 
1727 int mlxsw_sp_sb_occ_tc_port_bind_get(struct mlxsw_core_port *mlxsw_core_port,
1728 				     unsigned int sb_index, u16 tc_index,
1729 				     enum devlink_sb_pool_type pool_type,
1730 				     u32 *p_cur, u32 *p_max)
1731 {
1732 	struct mlxsw_sp_port *mlxsw_sp_port =
1733 			mlxsw_core_port_driver_priv(mlxsw_core_port);
1734 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1735 	u8 local_port = mlxsw_sp_port->local_port;
1736 	u8 pg_buff = tc_index;
1737 	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1738 	struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port,
1739 						       pg_buff, dir);
1740 
1741 	*p_cur = mlxsw_sp_cells_bytes(mlxsw_sp, cm->occ.cur);
1742 	*p_max = mlxsw_sp_cells_bytes(mlxsw_sp, cm->occ.max);
1743 	return 0;
1744 }
1745