1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/errno.h> 6 #include <linux/netdevice.h> 7 #include <net/pkt_cls.h> 8 #include <net/red.h> 9 10 #include "spectrum.h" 11 #include "spectrum_span.h" 12 #include "reg.h" 13 14 #define MLXSW_SP_PRIO_BAND_TO_TCLASS(band) (IEEE_8021QAZ_MAX_TCS - band - 1) 15 #define MLXSW_SP_PRIO_CHILD_TO_TCLASS(child) \ 16 MLXSW_SP_PRIO_BAND_TO_TCLASS((child - 1)) 17 18 enum mlxsw_sp_qdisc_type { 19 MLXSW_SP_QDISC_NO_QDISC, 20 MLXSW_SP_QDISC_RED, 21 MLXSW_SP_QDISC_PRIO, 22 MLXSW_SP_QDISC_ETS, 23 MLXSW_SP_QDISC_TBF, 24 MLXSW_SP_QDISC_FIFO, 25 }; 26 27 struct mlxsw_sp_qdisc; 28 29 struct mlxsw_sp_qdisc_ops { 30 enum mlxsw_sp_qdisc_type type; 31 int (*check_params)(struct mlxsw_sp_port *mlxsw_sp_port, 32 void *params); 33 int (*replace)(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, 34 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, void *params); 35 int (*destroy)(struct mlxsw_sp_port *mlxsw_sp_port, 36 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc); 37 int (*get_stats)(struct mlxsw_sp_port *mlxsw_sp_port, 38 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 39 struct tc_qopt_offload_stats *stats_ptr); 40 int (*get_xstats)(struct mlxsw_sp_port *mlxsw_sp_port, 41 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 42 void *xstats_ptr); 43 void (*clean_stats)(struct mlxsw_sp_port *mlxsw_sp_port, 44 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc); 45 /* unoffload - to be used for a qdisc that stops being offloaded without 46 * being destroyed. 47 */ 48 void (*unoffload)(struct mlxsw_sp_port *mlxsw_sp_port, 49 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, void *params); 50 struct mlxsw_sp_qdisc *(*find_class)(struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 51 u32 parent); 52 unsigned int num_classes; 53 }; 54 55 struct mlxsw_sp_qdisc { 56 u32 handle; 57 int tclass_num; 58 u8 prio_bitmap; 59 union { 60 struct red_stats red; 61 } xstats_base; 62 struct mlxsw_sp_qdisc_stats { 63 u64 tx_bytes; 64 u64 tx_packets; 65 u64 drops; 66 u64 overlimits; 67 u64 backlog; 68 } stats_base; 69 70 struct mlxsw_sp_qdisc_ops *ops; 71 struct mlxsw_sp_qdisc *parent; 72 struct mlxsw_sp_qdisc *qdiscs; 73 unsigned int num_classes; 74 }; 75 76 struct mlxsw_sp_qdisc_state { 77 struct mlxsw_sp_qdisc root_qdisc; 78 79 /* When a PRIO or ETS are added, the invisible FIFOs in their bands are 80 * created first. When notifications for these FIFOs arrive, it is not 81 * known what qdisc their parent handle refers to. It could be a 82 * newly-created PRIO that will replace the currently-offloaded one, or 83 * it could be e.g. a RED that will be attached below it. 84 * 85 * As the notifications start to arrive, use them to note what the 86 * future parent handle is, and keep track of which child FIFOs were 87 * seen. Then when the parent is known, retroactively offload those 88 * FIFOs. 89 */ 90 u32 future_handle; 91 bool future_fifos[IEEE_8021QAZ_MAX_TCS]; 92 struct mutex lock; /* Protects qdisc state. */ 93 }; 94 95 static bool 96 mlxsw_sp_qdisc_compare(struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, u32 handle) 97 { 98 return mlxsw_sp_qdisc->ops && mlxsw_sp_qdisc->handle == handle; 99 } 100 101 static struct mlxsw_sp_qdisc * 102 mlxsw_sp_qdisc_walk(struct mlxsw_sp_qdisc *qdisc, 103 struct mlxsw_sp_qdisc *(*pre)(struct mlxsw_sp_qdisc *, 104 void *), 105 void *data) 106 { 107 struct mlxsw_sp_qdisc *tmp; 108 unsigned int i; 109 110 if (pre) { 111 tmp = pre(qdisc, data); 112 if (tmp) 113 return tmp; 114 } 115 116 if (qdisc->ops) { 117 for (i = 0; i < qdisc->num_classes; i++) { 118 tmp = &qdisc->qdiscs[i]; 119 if (qdisc->ops) { 120 tmp = mlxsw_sp_qdisc_walk(tmp, pre, data); 121 if (tmp) 122 return tmp; 123 } 124 } 125 } 126 127 return NULL; 128 } 129 130 static struct mlxsw_sp_qdisc * 131 mlxsw_sp_qdisc_walk_cb_find(struct mlxsw_sp_qdisc *qdisc, void *data) 132 { 133 u32 parent = *(u32 *)data; 134 135 if (qdisc->ops && TC_H_MAJ(qdisc->handle) == TC_H_MAJ(parent)) { 136 if (qdisc->ops->find_class) 137 return qdisc->ops->find_class(qdisc, parent); 138 } 139 140 return NULL; 141 } 142 143 static struct mlxsw_sp_qdisc * 144 mlxsw_sp_qdisc_find(struct mlxsw_sp_port *mlxsw_sp_port, u32 parent, 145 bool root_only) 146 { 147 struct mlxsw_sp_qdisc_state *qdisc_state = mlxsw_sp_port->qdisc; 148 149 if (!qdisc_state) 150 return NULL; 151 if (parent == TC_H_ROOT) 152 return &qdisc_state->root_qdisc; 153 if (root_only) 154 return NULL; 155 return mlxsw_sp_qdisc_walk(&qdisc_state->root_qdisc, 156 mlxsw_sp_qdisc_walk_cb_find, &parent); 157 } 158 159 static struct mlxsw_sp_qdisc * 160 mlxsw_sp_qdisc_walk_cb_find_by_handle(struct mlxsw_sp_qdisc *qdisc, void *data) 161 { 162 u32 handle = *(u32 *)data; 163 164 if (qdisc->ops && qdisc->handle == handle) 165 return qdisc; 166 return NULL; 167 } 168 169 static struct mlxsw_sp_qdisc * 170 mlxsw_sp_qdisc_find_by_handle(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle) 171 { 172 struct mlxsw_sp_qdisc_state *qdisc_state = mlxsw_sp_port->qdisc; 173 174 if (!qdisc_state) 175 return NULL; 176 return mlxsw_sp_qdisc_walk(&qdisc_state->root_qdisc, 177 mlxsw_sp_qdisc_walk_cb_find_by_handle, 178 &handle); 179 } 180 181 static void 182 mlxsw_sp_qdisc_reduce_parent_backlog(struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 183 { 184 struct mlxsw_sp_qdisc *tmp; 185 186 for (tmp = mlxsw_sp_qdisc->parent; tmp; tmp = tmp->parent) 187 tmp->stats_base.backlog -= mlxsw_sp_qdisc->stats_base.backlog; 188 } 189 190 static int 191 mlxsw_sp_qdisc_destroy(struct mlxsw_sp_port *mlxsw_sp_port, 192 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 193 { 194 struct mlxsw_sp_qdisc *root_qdisc = &mlxsw_sp_port->qdisc->root_qdisc; 195 int err_hdroom = 0; 196 int err = 0; 197 198 if (!mlxsw_sp_qdisc) 199 return 0; 200 201 if (root_qdisc == mlxsw_sp_qdisc) { 202 struct mlxsw_sp_hdroom hdroom = *mlxsw_sp_port->hdroom; 203 204 hdroom.mode = MLXSW_SP_HDROOM_MODE_DCB; 205 mlxsw_sp_hdroom_prios_reset_buf_idx(&hdroom); 206 mlxsw_sp_hdroom_bufs_reset_lossiness(&hdroom); 207 mlxsw_sp_hdroom_bufs_reset_sizes(mlxsw_sp_port, &hdroom); 208 err_hdroom = mlxsw_sp_hdroom_configure(mlxsw_sp_port, &hdroom); 209 } 210 211 if (!mlxsw_sp_qdisc->ops) 212 return 0; 213 214 mlxsw_sp_qdisc_reduce_parent_backlog(mlxsw_sp_qdisc); 215 if (mlxsw_sp_qdisc->ops->destroy) 216 err = mlxsw_sp_qdisc->ops->destroy(mlxsw_sp_port, 217 mlxsw_sp_qdisc); 218 if (mlxsw_sp_qdisc->ops->clean_stats) 219 mlxsw_sp_qdisc->ops->clean_stats(mlxsw_sp_port, mlxsw_sp_qdisc); 220 221 mlxsw_sp_qdisc->handle = TC_H_UNSPEC; 222 mlxsw_sp_qdisc->ops = NULL; 223 mlxsw_sp_qdisc->num_classes = 0; 224 kfree(mlxsw_sp_qdisc->qdiscs); 225 mlxsw_sp_qdisc->qdiscs = NULL; 226 return err_hdroom ?: err; 227 } 228 229 static int mlxsw_sp_qdisc_create(struct mlxsw_sp_port *mlxsw_sp_port, 230 u32 handle, 231 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 232 struct mlxsw_sp_qdisc_ops *ops, void *params) 233 { 234 struct mlxsw_sp_qdisc *root_qdisc = &mlxsw_sp_port->qdisc->root_qdisc; 235 struct mlxsw_sp_hdroom orig_hdroom; 236 unsigned int i; 237 int err; 238 239 err = ops->check_params(mlxsw_sp_port, params); 240 if (err) 241 return err; 242 243 if (ops->num_classes) { 244 mlxsw_sp_qdisc->qdiscs = kcalloc(ops->num_classes, 245 sizeof(*mlxsw_sp_qdisc->qdiscs), 246 GFP_KERNEL); 247 if (!mlxsw_sp_qdisc->qdiscs) 248 return -ENOMEM; 249 250 for (i = 0; i < ops->num_classes; i++) 251 mlxsw_sp_qdisc->qdiscs[i].parent = mlxsw_sp_qdisc; 252 } 253 254 orig_hdroom = *mlxsw_sp_port->hdroom; 255 if (root_qdisc == mlxsw_sp_qdisc) { 256 struct mlxsw_sp_hdroom hdroom = orig_hdroom; 257 258 hdroom.mode = MLXSW_SP_HDROOM_MODE_TC; 259 mlxsw_sp_hdroom_prios_reset_buf_idx(&hdroom); 260 mlxsw_sp_hdroom_bufs_reset_lossiness(&hdroom); 261 mlxsw_sp_hdroom_bufs_reset_sizes(mlxsw_sp_port, &hdroom); 262 263 err = mlxsw_sp_hdroom_configure(mlxsw_sp_port, &hdroom); 264 if (err) 265 goto err_hdroom_configure; 266 } 267 268 mlxsw_sp_qdisc->num_classes = ops->num_classes; 269 mlxsw_sp_qdisc->ops = ops; 270 mlxsw_sp_qdisc->handle = handle; 271 err = ops->replace(mlxsw_sp_port, handle, mlxsw_sp_qdisc, params); 272 if (err) 273 goto err_replace; 274 275 return 0; 276 277 err_replace: 278 mlxsw_sp_qdisc->handle = TC_H_UNSPEC; 279 mlxsw_sp_qdisc->ops = NULL; 280 mlxsw_sp_qdisc->num_classes = 0; 281 mlxsw_sp_hdroom_configure(mlxsw_sp_port, &orig_hdroom); 282 err_hdroom_configure: 283 kfree(mlxsw_sp_qdisc->qdiscs); 284 mlxsw_sp_qdisc->qdiscs = NULL; 285 return err; 286 } 287 288 static int 289 mlxsw_sp_qdisc_change(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, 290 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, void *params) 291 { 292 struct mlxsw_sp_qdisc_ops *ops = mlxsw_sp_qdisc->ops; 293 int err; 294 295 err = ops->check_params(mlxsw_sp_port, params); 296 if (err) 297 goto unoffload; 298 299 err = ops->replace(mlxsw_sp_port, handle, mlxsw_sp_qdisc, params); 300 if (err) 301 goto unoffload; 302 303 /* Check if the Qdisc changed. That includes a situation where an 304 * invisible Qdisc replaces another one, or is being added for the 305 * first time. 306 */ 307 if (mlxsw_sp_qdisc->handle != handle) { 308 if (ops->clean_stats) 309 ops->clean_stats(mlxsw_sp_port, mlxsw_sp_qdisc); 310 } 311 312 mlxsw_sp_qdisc->handle = handle; 313 return 0; 314 315 unoffload: 316 if (ops->unoffload) 317 ops->unoffload(mlxsw_sp_port, mlxsw_sp_qdisc, params); 318 319 mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 320 return err; 321 } 322 323 static int 324 mlxsw_sp_qdisc_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, 325 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 326 struct mlxsw_sp_qdisc_ops *ops, void *params) 327 { 328 if (mlxsw_sp_qdisc->ops && mlxsw_sp_qdisc->ops->type != ops->type) 329 /* In case this location contained a different qdisc of the 330 * same type we can override the old qdisc configuration. 331 * Otherwise, we need to remove the old qdisc before setting the 332 * new one. 333 */ 334 mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 335 336 if (!mlxsw_sp_qdisc->ops) 337 return mlxsw_sp_qdisc_create(mlxsw_sp_port, handle, 338 mlxsw_sp_qdisc, ops, params); 339 else 340 return mlxsw_sp_qdisc_change(mlxsw_sp_port, handle, 341 mlxsw_sp_qdisc, params); 342 } 343 344 static int 345 mlxsw_sp_qdisc_get_stats(struct mlxsw_sp_port *mlxsw_sp_port, 346 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 347 struct tc_qopt_offload_stats *stats_ptr) 348 { 349 if (mlxsw_sp_qdisc && mlxsw_sp_qdisc->ops && 350 mlxsw_sp_qdisc->ops->get_stats) 351 return mlxsw_sp_qdisc->ops->get_stats(mlxsw_sp_port, 352 mlxsw_sp_qdisc, 353 stats_ptr); 354 355 return -EOPNOTSUPP; 356 } 357 358 static int 359 mlxsw_sp_qdisc_get_xstats(struct mlxsw_sp_port *mlxsw_sp_port, 360 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 361 void *xstats_ptr) 362 { 363 if (mlxsw_sp_qdisc && mlxsw_sp_qdisc->ops && 364 mlxsw_sp_qdisc->ops->get_xstats) 365 return mlxsw_sp_qdisc->ops->get_xstats(mlxsw_sp_port, 366 mlxsw_sp_qdisc, 367 xstats_ptr); 368 369 return -EOPNOTSUPP; 370 } 371 372 static u64 373 mlxsw_sp_xstats_backlog(struct mlxsw_sp_port_xstats *xstats, int tclass_num) 374 { 375 return xstats->backlog[tclass_num] + 376 xstats->backlog[tclass_num + 8]; 377 } 378 379 static u64 380 mlxsw_sp_xstats_tail_drop(struct mlxsw_sp_port_xstats *xstats, int tclass_num) 381 { 382 return xstats->tail_drop[tclass_num] + 383 xstats->tail_drop[tclass_num + 8]; 384 } 385 386 static void 387 mlxsw_sp_qdisc_bstats_per_priority_get(struct mlxsw_sp_port_xstats *xstats, 388 u8 prio_bitmap, u64 *tx_packets, 389 u64 *tx_bytes) 390 { 391 int i; 392 393 *tx_packets = 0; 394 *tx_bytes = 0; 395 for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { 396 if (prio_bitmap & BIT(i)) { 397 *tx_packets += xstats->tx_packets[i]; 398 *tx_bytes += xstats->tx_bytes[i]; 399 } 400 } 401 } 402 403 static void 404 mlxsw_sp_qdisc_collect_tc_stats(struct mlxsw_sp_port *mlxsw_sp_port, 405 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 406 u64 *p_tx_bytes, u64 *p_tx_packets, 407 u64 *p_drops, u64 *p_backlog) 408 { 409 int tclass_num = mlxsw_sp_qdisc->tclass_num; 410 struct mlxsw_sp_port_xstats *xstats; 411 u64 tx_bytes, tx_packets; 412 413 xstats = &mlxsw_sp_port->periodic_hw_stats.xstats; 414 mlxsw_sp_qdisc_bstats_per_priority_get(xstats, 415 mlxsw_sp_qdisc->prio_bitmap, 416 &tx_packets, &tx_bytes); 417 418 *p_tx_packets += tx_packets; 419 *p_tx_bytes += tx_bytes; 420 *p_drops += xstats->wred_drop[tclass_num] + 421 mlxsw_sp_xstats_tail_drop(xstats, tclass_num); 422 *p_backlog += mlxsw_sp_xstats_backlog(xstats, tclass_num); 423 } 424 425 static void 426 mlxsw_sp_qdisc_update_stats(struct mlxsw_sp *mlxsw_sp, 427 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 428 u64 tx_bytes, u64 tx_packets, 429 u64 drops, u64 backlog, 430 struct tc_qopt_offload_stats *stats_ptr) 431 { 432 struct mlxsw_sp_qdisc_stats *stats_base = &mlxsw_sp_qdisc->stats_base; 433 434 tx_bytes -= stats_base->tx_bytes; 435 tx_packets -= stats_base->tx_packets; 436 drops -= stats_base->drops; 437 backlog -= stats_base->backlog; 438 439 _bstats_update(stats_ptr->bstats, tx_bytes, tx_packets); 440 stats_ptr->qstats->drops += drops; 441 stats_ptr->qstats->backlog += mlxsw_sp_cells_bytes(mlxsw_sp, backlog); 442 443 stats_base->backlog += backlog; 444 stats_base->drops += drops; 445 stats_base->tx_bytes += tx_bytes; 446 stats_base->tx_packets += tx_packets; 447 } 448 449 static void 450 mlxsw_sp_qdisc_get_tc_stats(struct mlxsw_sp_port *mlxsw_sp_port, 451 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 452 struct tc_qopt_offload_stats *stats_ptr) 453 { 454 u64 tx_packets = 0; 455 u64 tx_bytes = 0; 456 u64 backlog = 0; 457 u64 drops = 0; 458 459 mlxsw_sp_qdisc_collect_tc_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 460 &tx_bytes, &tx_packets, 461 &drops, &backlog); 462 mlxsw_sp_qdisc_update_stats(mlxsw_sp_port->mlxsw_sp, mlxsw_sp_qdisc, 463 tx_bytes, tx_packets, drops, backlog, 464 stats_ptr); 465 } 466 467 static int 468 mlxsw_sp_tclass_congestion_enable(struct mlxsw_sp_port *mlxsw_sp_port, 469 int tclass_num, u32 min, u32 max, 470 u32 probability, bool is_wred, bool is_ecn) 471 { 472 char cwtpm_cmd[MLXSW_REG_CWTPM_LEN]; 473 char cwtp_cmd[MLXSW_REG_CWTP_LEN]; 474 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 475 int err; 476 477 mlxsw_reg_cwtp_pack(cwtp_cmd, mlxsw_sp_port->local_port, tclass_num); 478 mlxsw_reg_cwtp_profile_pack(cwtp_cmd, MLXSW_REG_CWTP_DEFAULT_PROFILE, 479 roundup(min, MLXSW_REG_CWTP_MIN_VALUE), 480 roundup(max, MLXSW_REG_CWTP_MIN_VALUE), 481 probability); 482 483 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(cwtp), cwtp_cmd); 484 if (err) 485 return err; 486 487 mlxsw_reg_cwtpm_pack(cwtpm_cmd, mlxsw_sp_port->local_port, tclass_num, 488 MLXSW_REG_CWTP_DEFAULT_PROFILE, is_wred, is_ecn); 489 490 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(cwtpm), cwtpm_cmd); 491 } 492 493 static int 494 mlxsw_sp_tclass_congestion_disable(struct mlxsw_sp_port *mlxsw_sp_port, 495 int tclass_num) 496 { 497 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 498 char cwtpm_cmd[MLXSW_REG_CWTPM_LEN]; 499 500 mlxsw_reg_cwtpm_pack(cwtpm_cmd, mlxsw_sp_port->local_port, tclass_num, 501 MLXSW_REG_CWTPM_RESET_PROFILE, false, false); 502 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(cwtpm), cwtpm_cmd); 503 } 504 505 static void 506 mlxsw_sp_setup_tc_qdisc_red_clean_stats(struct mlxsw_sp_port *mlxsw_sp_port, 507 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 508 { 509 int tclass_num = mlxsw_sp_qdisc->tclass_num; 510 struct mlxsw_sp_qdisc_stats *stats_base; 511 struct mlxsw_sp_port_xstats *xstats; 512 struct red_stats *red_base; 513 514 xstats = &mlxsw_sp_port->periodic_hw_stats.xstats; 515 stats_base = &mlxsw_sp_qdisc->stats_base; 516 red_base = &mlxsw_sp_qdisc->xstats_base.red; 517 518 mlxsw_sp_qdisc_bstats_per_priority_get(xstats, 519 mlxsw_sp_qdisc->prio_bitmap, 520 &stats_base->tx_packets, 521 &stats_base->tx_bytes); 522 red_base->prob_mark = xstats->tc_ecn[tclass_num]; 523 red_base->prob_drop = xstats->wred_drop[tclass_num]; 524 red_base->pdrop = mlxsw_sp_xstats_tail_drop(xstats, tclass_num); 525 526 stats_base->overlimits = red_base->prob_drop + red_base->prob_mark; 527 stats_base->drops = red_base->prob_drop + red_base->pdrop; 528 529 stats_base->backlog = 0; 530 } 531 532 static int 533 mlxsw_sp_qdisc_red_destroy(struct mlxsw_sp_port *mlxsw_sp_port, 534 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 535 { 536 return mlxsw_sp_tclass_congestion_disable(mlxsw_sp_port, 537 mlxsw_sp_qdisc->tclass_num); 538 } 539 540 static int 541 mlxsw_sp_qdisc_red_check_params(struct mlxsw_sp_port *mlxsw_sp_port, 542 void *params) 543 { 544 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 545 struct tc_red_qopt_offload_params *p = params; 546 547 if (p->min > p->max) { 548 dev_err(mlxsw_sp->bus_info->dev, 549 "spectrum: RED: min %u is bigger then max %u\n", p->min, 550 p->max); 551 return -EINVAL; 552 } 553 if (p->max > MLXSW_CORE_RES_GET(mlxsw_sp->core, 554 GUARANTEED_SHARED_BUFFER)) { 555 dev_err(mlxsw_sp->bus_info->dev, 556 "spectrum: RED: max value %u is too big\n", p->max); 557 return -EINVAL; 558 } 559 if (p->min == 0 || p->max == 0) { 560 dev_err(mlxsw_sp->bus_info->dev, 561 "spectrum: RED: 0 value is illegal for min and max\n"); 562 return -EINVAL; 563 } 564 return 0; 565 } 566 567 static int 568 mlxsw_sp_qdisc_red_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, 569 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 570 void *params) 571 { 572 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 573 struct tc_red_qopt_offload_params *p = params; 574 int tclass_num = mlxsw_sp_qdisc->tclass_num; 575 u32 min, max; 576 u64 prob; 577 578 /* calculate probability in percentage */ 579 prob = p->probability; 580 prob *= 100; 581 prob = DIV_ROUND_UP(prob, 1 << 16); 582 prob = DIV_ROUND_UP(prob, 1 << 16); 583 min = mlxsw_sp_bytes_cells(mlxsw_sp, p->min); 584 max = mlxsw_sp_bytes_cells(mlxsw_sp, p->max); 585 return mlxsw_sp_tclass_congestion_enable(mlxsw_sp_port, tclass_num, 586 min, max, prob, 587 !p->is_nodrop, p->is_ecn); 588 } 589 590 static void 591 mlxsw_sp_qdisc_leaf_unoffload(struct mlxsw_sp_port *mlxsw_sp_port, 592 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 593 struct gnet_stats_queue *qstats) 594 { 595 u64 backlog; 596 597 backlog = mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp, 598 mlxsw_sp_qdisc->stats_base.backlog); 599 qstats->backlog -= backlog; 600 mlxsw_sp_qdisc->stats_base.backlog = 0; 601 } 602 603 static void 604 mlxsw_sp_qdisc_red_unoffload(struct mlxsw_sp_port *mlxsw_sp_port, 605 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 606 void *params) 607 { 608 struct tc_red_qopt_offload_params *p = params; 609 610 mlxsw_sp_qdisc_leaf_unoffload(mlxsw_sp_port, mlxsw_sp_qdisc, p->qstats); 611 } 612 613 static int 614 mlxsw_sp_qdisc_get_red_xstats(struct mlxsw_sp_port *mlxsw_sp_port, 615 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 616 void *xstats_ptr) 617 { 618 struct red_stats *xstats_base = &mlxsw_sp_qdisc->xstats_base.red; 619 int tclass_num = mlxsw_sp_qdisc->tclass_num; 620 struct mlxsw_sp_port_xstats *xstats; 621 struct red_stats *res = xstats_ptr; 622 int early_drops, marks, pdrops; 623 624 xstats = &mlxsw_sp_port->periodic_hw_stats.xstats; 625 626 early_drops = xstats->wred_drop[tclass_num] - xstats_base->prob_drop; 627 marks = xstats->tc_ecn[tclass_num] - xstats_base->prob_mark; 628 pdrops = mlxsw_sp_xstats_tail_drop(xstats, tclass_num) - 629 xstats_base->pdrop; 630 631 res->pdrop += pdrops; 632 res->prob_drop += early_drops; 633 res->prob_mark += marks; 634 635 xstats_base->pdrop += pdrops; 636 xstats_base->prob_drop += early_drops; 637 xstats_base->prob_mark += marks; 638 return 0; 639 } 640 641 static int 642 mlxsw_sp_qdisc_get_red_stats(struct mlxsw_sp_port *mlxsw_sp_port, 643 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 644 struct tc_qopt_offload_stats *stats_ptr) 645 { 646 int tclass_num = mlxsw_sp_qdisc->tclass_num; 647 struct mlxsw_sp_qdisc_stats *stats_base; 648 struct mlxsw_sp_port_xstats *xstats; 649 u64 overlimits; 650 651 xstats = &mlxsw_sp_port->periodic_hw_stats.xstats; 652 stats_base = &mlxsw_sp_qdisc->stats_base; 653 654 mlxsw_sp_qdisc_get_tc_stats(mlxsw_sp_port, mlxsw_sp_qdisc, stats_ptr); 655 overlimits = xstats->wred_drop[tclass_num] + 656 xstats->tc_ecn[tclass_num] - stats_base->overlimits; 657 658 stats_ptr->qstats->overlimits += overlimits; 659 stats_base->overlimits += overlimits; 660 661 return 0; 662 } 663 664 static struct mlxsw_sp_qdisc * 665 mlxsw_sp_qdisc_leaf_find_class(struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 666 u32 parent) 667 { 668 return NULL; 669 } 670 671 #define MLXSW_SP_PORT_DEFAULT_TCLASS 0 672 673 static struct mlxsw_sp_qdisc_ops mlxsw_sp_qdisc_ops_red = { 674 .type = MLXSW_SP_QDISC_RED, 675 .check_params = mlxsw_sp_qdisc_red_check_params, 676 .replace = mlxsw_sp_qdisc_red_replace, 677 .unoffload = mlxsw_sp_qdisc_red_unoffload, 678 .destroy = mlxsw_sp_qdisc_red_destroy, 679 .get_stats = mlxsw_sp_qdisc_get_red_stats, 680 .get_xstats = mlxsw_sp_qdisc_get_red_xstats, 681 .clean_stats = mlxsw_sp_setup_tc_qdisc_red_clean_stats, 682 .find_class = mlxsw_sp_qdisc_leaf_find_class, 683 }; 684 685 static int __mlxsw_sp_setup_tc_red(struct mlxsw_sp_port *mlxsw_sp_port, 686 struct tc_red_qopt_offload *p) 687 { 688 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc; 689 690 mlxsw_sp_qdisc = mlxsw_sp_qdisc_find(mlxsw_sp_port, p->parent, false); 691 if (!mlxsw_sp_qdisc) 692 return -EOPNOTSUPP; 693 694 if (p->command == TC_RED_REPLACE) 695 return mlxsw_sp_qdisc_replace(mlxsw_sp_port, p->handle, 696 mlxsw_sp_qdisc, 697 &mlxsw_sp_qdisc_ops_red, 698 &p->set); 699 700 if (!mlxsw_sp_qdisc_compare(mlxsw_sp_qdisc, p->handle)) 701 return -EOPNOTSUPP; 702 703 switch (p->command) { 704 case TC_RED_DESTROY: 705 return mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 706 case TC_RED_XSTATS: 707 return mlxsw_sp_qdisc_get_xstats(mlxsw_sp_port, mlxsw_sp_qdisc, 708 p->xstats); 709 case TC_RED_STATS: 710 return mlxsw_sp_qdisc_get_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 711 &p->stats); 712 default: 713 return -EOPNOTSUPP; 714 } 715 } 716 717 int mlxsw_sp_setup_tc_red(struct mlxsw_sp_port *mlxsw_sp_port, 718 struct tc_red_qopt_offload *p) 719 { 720 int err; 721 722 mutex_lock(&mlxsw_sp_port->qdisc->lock); 723 err = __mlxsw_sp_setup_tc_red(mlxsw_sp_port, p); 724 mutex_unlock(&mlxsw_sp_port->qdisc->lock); 725 726 return err; 727 } 728 729 static void 730 mlxsw_sp_setup_tc_qdisc_leaf_clean_stats(struct mlxsw_sp_port *mlxsw_sp_port, 731 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 732 { 733 u64 backlog_cells = 0; 734 u64 tx_packets = 0; 735 u64 tx_bytes = 0; 736 u64 drops = 0; 737 738 mlxsw_sp_qdisc_collect_tc_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 739 &tx_bytes, &tx_packets, 740 &drops, &backlog_cells); 741 742 mlxsw_sp_qdisc->stats_base.tx_packets = tx_packets; 743 mlxsw_sp_qdisc->stats_base.tx_bytes = tx_bytes; 744 mlxsw_sp_qdisc->stats_base.drops = drops; 745 mlxsw_sp_qdisc->stats_base.backlog = 0; 746 } 747 748 static int 749 mlxsw_sp_qdisc_tbf_destroy(struct mlxsw_sp_port *mlxsw_sp_port, 750 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 751 { 752 return mlxsw_sp_port_ets_maxrate_set(mlxsw_sp_port, 753 MLXSW_REG_QEEC_HR_SUBGROUP, 754 mlxsw_sp_qdisc->tclass_num, 0, 755 MLXSW_REG_QEEC_MAS_DIS, 0); 756 } 757 758 static int 759 mlxsw_sp_qdisc_tbf_bs(struct mlxsw_sp_port *mlxsw_sp_port, 760 u32 max_size, u8 *p_burst_size) 761 { 762 /* TBF burst size is configured in bytes. The ASIC burst size value is 763 * ((2 ^ bs) * 512 bits. Convert the TBF bytes to 512-bit units. 764 */ 765 u32 bs512 = max_size / 64; 766 u8 bs = fls(bs512); 767 768 if (!bs) 769 return -EINVAL; 770 --bs; 771 772 /* Demand a power of two. */ 773 if ((1 << bs) != bs512) 774 return -EINVAL; 775 776 if (bs < mlxsw_sp_port->mlxsw_sp->lowest_shaper_bs || 777 bs > MLXSW_REG_QEEC_HIGHEST_SHAPER_BS) 778 return -EINVAL; 779 780 *p_burst_size = bs; 781 return 0; 782 } 783 784 static u32 785 mlxsw_sp_qdisc_tbf_max_size(u8 bs) 786 { 787 return (1U << bs) * 64; 788 } 789 790 static u64 791 mlxsw_sp_qdisc_tbf_rate_kbps(struct tc_tbf_qopt_offload_replace_params *p) 792 { 793 /* TBF interface is in bytes/s, whereas Spectrum ASIC is configured in 794 * Kbits/s. 795 */ 796 return div_u64(p->rate.rate_bytes_ps, 1000) * 8; 797 } 798 799 static int 800 mlxsw_sp_qdisc_tbf_check_params(struct mlxsw_sp_port *mlxsw_sp_port, 801 void *params) 802 { 803 struct tc_tbf_qopt_offload_replace_params *p = params; 804 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 805 u64 rate_kbps = mlxsw_sp_qdisc_tbf_rate_kbps(p); 806 u8 burst_size; 807 int err; 808 809 if (rate_kbps >= MLXSW_REG_QEEC_MAS_DIS) { 810 dev_err(mlxsw_sp_port->mlxsw_sp->bus_info->dev, 811 "spectrum: TBF: rate of %lluKbps must be below %u\n", 812 rate_kbps, MLXSW_REG_QEEC_MAS_DIS); 813 return -EINVAL; 814 } 815 816 err = mlxsw_sp_qdisc_tbf_bs(mlxsw_sp_port, p->max_size, &burst_size); 817 if (err) { 818 u8 highest_shaper_bs = MLXSW_REG_QEEC_HIGHEST_SHAPER_BS; 819 820 dev_err(mlxsw_sp->bus_info->dev, 821 "spectrum: TBF: invalid burst size of %u, must be a power of two between %u and %u", 822 p->max_size, 823 mlxsw_sp_qdisc_tbf_max_size(mlxsw_sp->lowest_shaper_bs), 824 mlxsw_sp_qdisc_tbf_max_size(highest_shaper_bs)); 825 return -EINVAL; 826 } 827 828 return 0; 829 } 830 831 static int 832 mlxsw_sp_qdisc_tbf_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, 833 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 834 void *params) 835 { 836 struct tc_tbf_qopt_offload_replace_params *p = params; 837 u64 rate_kbps = mlxsw_sp_qdisc_tbf_rate_kbps(p); 838 u8 burst_size; 839 int err; 840 841 err = mlxsw_sp_qdisc_tbf_bs(mlxsw_sp_port, p->max_size, &burst_size); 842 if (WARN_ON_ONCE(err)) 843 /* check_params above was supposed to reject this value. */ 844 return -EINVAL; 845 846 /* Configure subgroup shaper, so that both UC and MC traffic is subject 847 * to shaping. That is unlike RED, however UC queue lengths are going to 848 * be different than MC ones due to different pool and quota 849 * configurations, so the configuration is not applicable. For shaper on 850 * the other hand, subjecting the overall stream to the configured 851 * shaper makes sense. Also note that that is what we do for 852 * ieee_setmaxrate(). 853 */ 854 return mlxsw_sp_port_ets_maxrate_set(mlxsw_sp_port, 855 MLXSW_REG_QEEC_HR_SUBGROUP, 856 mlxsw_sp_qdisc->tclass_num, 0, 857 rate_kbps, burst_size); 858 } 859 860 static void 861 mlxsw_sp_qdisc_tbf_unoffload(struct mlxsw_sp_port *mlxsw_sp_port, 862 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 863 void *params) 864 { 865 struct tc_tbf_qopt_offload_replace_params *p = params; 866 867 mlxsw_sp_qdisc_leaf_unoffload(mlxsw_sp_port, mlxsw_sp_qdisc, p->qstats); 868 } 869 870 static int 871 mlxsw_sp_qdisc_get_tbf_stats(struct mlxsw_sp_port *mlxsw_sp_port, 872 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 873 struct tc_qopt_offload_stats *stats_ptr) 874 { 875 mlxsw_sp_qdisc_get_tc_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 876 stats_ptr); 877 return 0; 878 } 879 880 static struct mlxsw_sp_qdisc_ops mlxsw_sp_qdisc_ops_tbf = { 881 .type = MLXSW_SP_QDISC_TBF, 882 .check_params = mlxsw_sp_qdisc_tbf_check_params, 883 .replace = mlxsw_sp_qdisc_tbf_replace, 884 .unoffload = mlxsw_sp_qdisc_tbf_unoffload, 885 .destroy = mlxsw_sp_qdisc_tbf_destroy, 886 .get_stats = mlxsw_sp_qdisc_get_tbf_stats, 887 .clean_stats = mlxsw_sp_setup_tc_qdisc_leaf_clean_stats, 888 .find_class = mlxsw_sp_qdisc_leaf_find_class, 889 }; 890 891 static int __mlxsw_sp_setup_tc_tbf(struct mlxsw_sp_port *mlxsw_sp_port, 892 struct tc_tbf_qopt_offload *p) 893 { 894 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc; 895 896 mlxsw_sp_qdisc = mlxsw_sp_qdisc_find(mlxsw_sp_port, p->parent, false); 897 if (!mlxsw_sp_qdisc) 898 return -EOPNOTSUPP; 899 900 if (p->command == TC_TBF_REPLACE) 901 return mlxsw_sp_qdisc_replace(mlxsw_sp_port, p->handle, 902 mlxsw_sp_qdisc, 903 &mlxsw_sp_qdisc_ops_tbf, 904 &p->replace_params); 905 906 if (!mlxsw_sp_qdisc_compare(mlxsw_sp_qdisc, p->handle)) 907 return -EOPNOTSUPP; 908 909 switch (p->command) { 910 case TC_TBF_DESTROY: 911 return mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 912 case TC_TBF_STATS: 913 return mlxsw_sp_qdisc_get_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 914 &p->stats); 915 default: 916 return -EOPNOTSUPP; 917 } 918 } 919 920 int mlxsw_sp_setup_tc_tbf(struct mlxsw_sp_port *mlxsw_sp_port, 921 struct tc_tbf_qopt_offload *p) 922 { 923 int err; 924 925 mutex_lock(&mlxsw_sp_port->qdisc->lock); 926 err = __mlxsw_sp_setup_tc_tbf(mlxsw_sp_port, p); 927 mutex_unlock(&mlxsw_sp_port->qdisc->lock); 928 929 return err; 930 } 931 932 static int 933 mlxsw_sp_qdisc_fifo_check_params(struct mlxsw_sp_port *mlxsw_sp_port, 934 void *params) 935 { 936 return 0; 937 } 938 939 static int 940 mlxsw_sp_qdisc_fifo_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, 941 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 942 void *params) 943 { 944 return 0; 945 } 946 947 static int 948 mlxsw_sp_qdisc_get_fifo_stats(struct mlxsw_sp_port *mlxsw_sp_port, 949 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 950 struct tc_qopt_offload_stats *stats_ptr) 951 { 952 mlxsw_sp_qdisc_get_tc_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 953 stats_ptr); 954 return 0; 955 } 956 957 static struct mlxsw_sp_qdisc_ops mlxsw_sp_qdisc_ops_fifo = { 958 .type = MLXSW_SP_QDISC_FIFO, 959 .check_params = mlxsw_sp_qdisc_fifo_check_params, 960 .replace = mlxsw_sp_qdisc_fifo_replace, 961 .get_stats = mlxsw_sp_qdisc_get_fifo_stats, 962 .clean_stats = mlxsw_sp_setup_tc_qdisc_leaf_clean_stats, 963 }; 964 965 static int __mlxsw_sp_setup_tc_fifo(struct mlxsw_sp_port *mlxsw_sp_port, 966 struct tc_fifo_qopt_offload *p) 967 { 968 struct mlxsw_sp_qdisc_state *qdisc_state = mlxsw_sp_port->qdisc; 969 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc; 970 unsigned int band; 971 u32 parent_handle; 972 973 mlxsw_sp_qdisc = mlxsw_sp_qdisc_find(mlxsw_sp_port, p->parent, false); 974 if (!mlxsw_sp_qdisc && p->handle == TC_H_UNSPEC) { 975 parent_handle = TC_H_MAJ(p->parent); 976 if (parent_handle != qdisc_state->future_handle) { 977 /* This notifications is for a different Qdisc than 978 * previously. Wipe the future cache. 979 */ 980 memset(qdisc_state->future_fifos, 0, 981 sizeof(qdisc_state->future_fifos)); 982 qdisc_state->future_handle = parent_handle; 983 } 984 985 band = TC_H_MIN(p->parent) - 1; 986 if (band < IEEE_8021QAZ_MAX_TCS) { 987 if (p->command == TC_FIFO_REPLACE) 988 qdisc_state->future_fifos[band] = true; 989 else if (p->command == TC_FIFO_DESTROY) 990 qdisc_state->future_fifos[band] = false; 991 } 992 } 993 if (!mlxsw_sp_qdisc) 994 return -EOPNOTSUPP; 995 996 if (p->command == TC_FIFO_REPLACE) { 997 return mlxsw_sp_qdisc_replace(mlxsw_sp_port, p->handle, 998 mlxsw_sp_qdisc, 999 &mlxsw_sp_qdisc_ops_fifo, NULL); 1000 } 1001 1002 if (!mlxsw_sp_qdisc_compare(mlxsw_sp_qdisc, p->handle)) 1003 return -EOPNOTSUPP; 1004 1005 switch (p->command) { 1006 case TC_FIFO_DESTROY: 1007 return mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 1008 case TC_FIFO_STATS: 1009 return mlxsw_sp_qdisc_get_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 1010 &p->stats); 1011 case TC_FIFO_REPLACE: /* Handled above. */ 1012 break; 1013 } 1014 1015 return -EOPNOTSUPP; 1016 } 1017 1018 int mlxsw_sp_setup_tc_fifo(struct mlxsw_sp_port *mlxsw_sp_port, 1019 struct tc_fifo_qopt_offload *p) 1020 { 1021 int err; 1022 1023 mutex_lock(&mlxsw_sp_port->qdisc->lock); 1024 err = __mlxsw_sp_setup_tc_fifo(mlxsw_sp_port, p); 1025 mutex_unlock(&mlxsw_sp_port->qdisc->lock); 1026 1027 return err; 1028 } 1029 1030 static int __mlxsw_sp_qdisc_ets_destroy(struct mlxsw_sp_port *mlxsw_sp_port, 1031 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 1032 { 1033 int i; 1034 1035 for (i = 0; i < mlxsw_sp_qdisc->num_classes; i++) { 1036 mlxsw_sp_port_prio_tc_set(mlxsw_sp_port, i, 1037 MLXSW_SP_PORT_DEFAULT_TCLASS); 1038 mlxsw_sp_port_ets_set(mlxsw_sp_port, 1039 MLXSW_REG_QEEC_HR_SUBGROUP, 1040 i, 0, false, 0); 1041 mlxsw_sp_qdisc_destroy(mlxsw_sp_port, 1042 &mlxsw_sp_qdisc->qdiscs[i]); 1043 mlxsw_sp_qdisc->qdiscs[i].prio_bitmap = 0; 1044 } 1045 1046 return 0; 1047 } 1048 1049 static int 1050 mlxsw_sp_qdisc_prio_destroy(struct mlxsw_sp_port *mlxsw_sp_port, 1051 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 1052 { 1053 return __mlxsw_sp_qdisc_ets_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 1054 } 1055 1056 static int 1057 __mlxsw_sp_qdisc_ets_check_params(unsigned int nbands) 1058 { 1059 if (nbands > IEEE_8021QAZ_MAX_TCS) 1060 return -EOPNOTSUPP; 1061 1062 return 0; 1063 } 1064 1065 static int 1066 mlxsw_sp_qdisc_prio_check_params(struct mlxsw_sp_port *mlxsw_sp_port, 1067 void *params) 1068 { 1069 struct tc_prio_qopt_offload_params *p = params; 1070 1071 return __mlxsw_sp_qdisc_ets_check_params(p->bands); 1072 } 1073 1074 static int 1075 __mlxsw_sp_qdisc_ets_replace(struct mlxsw_sp_port *mlxsw_sp_port, 1076 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1077 u32 handle, unsigned int nbands, 1078 const unsigned int *quanta, 1079 const unsigned int *weights, 1080 const u8 *priomap) 1081 { 1082 struct mlxsw_sp_qdisc_state *qdisc_state = mlxsw_sp_port->qdisc; 1083 struct mlxsw_sp_qdisc *child_qdisc; 1084 int tclass, i, band, backlog; 1085 u8 old_priomap; 1086 int err; 1087 1088 for (band = 0; band < nbands; band++) { 1089 tclass = MLXSW_SP_PRIO_BAND_TO_TCLASS(band); 1090 child_qdisc = &mlxsw_sp_qdisc->qdiscs[band]; 1091 old_priomap = child_qdisc->prio_bitmap; 1092 child_qdisc->prio_bitmap = 0; 1093 1094 err = mlxsw_sp_port_ets_set(mlxsw_sp_port, 1095 MLXSW_REG_QEEC_HR_SUBGROUP, 1096 tclass, 0, !!quanta[band], 1097 weights[band]); 1098 if (err) 1099 return err; 1100 1101 for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { 1102 if (priomap[i] == band) { 1103 child_qdisc->prio_bitmap |= BIT(i); 1104 if (BIT(i) & old_priomap) 1105 continue; 1106 err = mlxsw_sp_port_prio_tc_set(mlxsw_sp_port, 1107 i, tclass); 1108 if (err) 1109 return err; 1110 } 1111 } 1112 1113 child_qdisc->tclass_num = tclass; 1114 1115 if (old_priomap != child_qdisc->prio_bitmap && 1116 child_qdisc->ops && child_qdisc->ops->clean_stats) { 1117 backlog = child_qdisc->stats_base.backlog; 1118 child_qdisc->ops->clean_stats(mlxsw_sp_port, 1119 child_qdisc); 1120 child_qdisc->stats_base.backlog = backlog; 1121 } 1122 1123 if (handle == qdisc_state->future_handle && 1124 qdisc_state->future_fifos[band]) { 1125 err = mlxsw_sp_qdisc_replace(mlxsw_sp_port, TC_H_UNSPEC, 1126 child_qdisc, 1127 &mlxsw_sp_qdisc_ops_fifo, 1128 NULL); 1129 if (err) 1130 return err; 1131 } 1132 } 1133 for (; band < IEEE_8021QAZ_MAX_TCS; band++) { 1134 tclass = MLXSW_SP_PRIO_BAND_TO_TCLASS(band); 1135 child_qdisc = &mlxsw_sp_qdisc->qdiscs[band]; 1136 child_qdisc->prio_bitmap = 0; 1137 mlxsw_sp_qdisc_destroy(mlxsw_sp_port, child_qdisc); 1138 mlxsw_sp_port_ets_set(mlxsw_sp_port, 1139 MLXSW_REG_QEEC_HR_SUBGROUP, 1140 tclass, 0, false, 0); 1141 } 1142 1143 qdisc_state->future_handle = TC_H_UNSPEC; 1144 memset(qdisc_state->future_fifos, 0, sizeof(qdisc_state->future_fifos)); 1145 return 0; 1146 } 1147 1148 static int 1149 mlxsw_sp_qdisc_prio_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, 1150 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1151 void *params) 1152 { 1153 struct tc_prio_qopt_offload_params *p = params; 1154 unsigned int zeroes[TCQ_ETS_MAX_BANDS] = {0}; 1155 1156 return __mlxsw_sp_qdisc_ets_replace(mlxsw_sp_port, mlxsw_sp_qdisc, 1157 handle, p->bands, zeroes, 1158 zeroes, p->priomap); 1159 } 1160 1161 static void 1162 __mlxsw_sp_qdisc_ets_unoffload(struct mlxsw_sp_port *mlxsw_sp_port, 1163 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1164 struct gnet_stats_queue *qstats) 1165 { 1166 u64 backlog; 1167 1168 backlog = mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp, 1169 mlxsw_sp_qdisc->stats_base.backlog); 1170 qstats->backlog -= backlog; 1171 } 1172 1173 static void 1174 mlxsw_sp_qdisc_prio_unoffload(struct mlxsw_sp_port *mlxsw_sp_port, 1175 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1176 void *params) 1177 { 1178 struct tc_prio_qopt_offload_params *p = params; 1179 1180 __mlxsw_sp_qdisc_ets_unoffload(mlxsw_sp_port, mlxsw_sp_qdisc, 1181 p->qstats); 1182 } 1183 1184 static int 1185 mlxsw_sp_qdisc_get_prio_stats(struct mlxsw_sp_port *mlxsw_sp_port, 1186 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1187 struct tc_qopt_offload_stats *stats_ptr) 1188 { 1189 struct mlxsw_sp_qdisc *tc_qdisc; 1190 u64 tx_packets = 0; 1191 u64 tx_bytes = 0; 1192 u64 backlog = 0; 1193 u64 drops = 0; 1194 int i; 1195 1196 for (i = 0; i < mlxsw_sp_qdisc->num_classes; i++) { 1197 tc_qdisc = &mlxsw_sp_qdisc->qdiscs[i]; 1198 mlxsw_sp_qdisc_collect_tc_stats(mlxsw_sp_port, tc_qdisc, 1199 &tx_bytes, &tx_packets, 1200 &drops, &backlog); 1201 } 1202 1203 mlxsw_sp_qdisc_update_stats(mlxsw_sp_port->mlxsw_sp, mlxsw_sp_qdisc, 1204 tx_bytes, tx_packets, drops, backlog, 1205 stats_ptr); 1206 return 0; 1207 } 1208 1209 static void 1210 mlxsw_sp_setup_tc_qdisc_prio_clean_stats(struct mlxsw_sp_port *mlxsw_sp_port, 1211 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 1212 { 1213 struct mlxsw_sp_qdisc_stats *stats_base; 1214 struct mlxsw_sp_port_xstats *xstats; 1215 struct rtnl_link_stats64 *stats; 1216 int i; 1217 1218 xstats = &mlxsw_sp_port->periodic_hw_stats.xstats; 1219 stats = &mlxsw_sp_port->periodic_hw_stats.stats; 1220 stats_base = &mlxsw_sp_qdisc->stats_base; 1221 1222 stats_base->tx_packets = stats->tx_packets; 1223 stats_base->tx_bytes = stats->tx_bytes; 1224 1225 stats_base->drops = 0; 1226 for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { 1227 stats_base->drops += mlxsw_sp_xstats_tail_drop(xstats, i); 1228 stats_base->drops += xstats->wred_drop[i]; 1229 } 1230 1231 mlxsw_sp_qdisc->stats_base.backlog = 0; 1232 } 1233 1234 static struct mlxsw_sp_qdisc * 1235 mlxsw_sp_qdisc_prio_find_class(struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1236 u32 parent) 1237 { 1238 int child_index = TC_H_MIN(parent); 1239 int band = child_index - 1; 1240 1241 if (band < 0 || band >= mlxsw_sp_qdisc->num_classes) 1242 return NULL; 1243 return &mlxsw_sp_qdisc->qdiscs[band]; 1244 } 1245 1246 static struct mlxsw_sp_qdisc_ops mlxsw_sp_qdisc_ops_prio = { 1247 .type = MLXSW_SP_QDISC_PRIO, 1248 .check_params = mlxsw_sp_qdisc_prio_check_params, 1249 .replace = mlxsw_sp_qdisc_prio_replace, 1250 .unoffload = mlxsw_sp_qdisc_prio_unoffload, 1251 .destroy = mlxsw_sp_qdisc_prio_destroy, 1252 .get_stats = mlxsw_sp_qdisc_get_prio_stats, 1253 .clean_stats = mlxsw_sp_setup_tc_qdisc_prio_clean_stats, 1254 .find_class = mlxsw_sp_qdisc_prio_find_class, 1255 .num_classes = IEEE_8021QAZ_MAX_TCS, 1256 }; 1257 1258 static int 1259 mlxsw_sp_qdisc_ets_check_params(struct mlxsw_sp_port *mlxsw_sp_port, 1260 void *params) 1261 { 1262 struct tc_ets_qopt_offload_replace_params *p = params; 1263 1264 return __mlxsw_sp_qdisc_ets_check_params(p->bands); 1265 } 1266 1267 static int 1268 mlxsw_sp_qdisc_ets_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, 1269 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1270 void *params) 1271 { 1272 struct tc_ets_qopt_offload_replace_params *p = params; 1273 1274 return __mlxsw_sp_qdisc_ets_replace(mlxsw_sp_port, mlxsw_sp_qdisc, 1275 handle, p->bands, p->quanta, 1276 p->weights, p->priomap); 1277 } 1278 1279 static void 1280 mlxsw_sp_qdisc_ets_unoffload(struct mlxsw_sp_port *mlxsw_sp_port, 1281 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1282 void *params) 1283 { 1284 struct tc_ets_qopt_offload_replace_params *p = params; 1285 1286 __mlxsw_sp_qdisc_ets_unoffload(mlxsw_sp_port, mlxsw_sp_qdisc, 1287 p->qstats); 1288 } 1289 1290 static int 1291 mlxsw_sp_qdisc_ets_destroy(struct mlxsw_sp_port *mlxsw_sp_port, 1292 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc) 1293 { 1294 return __mlxsw_sp_qdisc_ets_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 1295 } 1296 1297 static struct mlxsw_sp_qdisc_ops mlxsw_sp_qdisc_ops_ets = { 1298 .type = MLXSW_SP_QDISC_ETS, 1299 .check_params = mlxsw_sp_qdisc_ets_check_params, 1300 .replace = mlxsw_sp_qdisc_ets_replace, 1301 .unoffload = mlxsw_sp_qdisc_ets_unoffload, 1302 .destroy = mlxsw_sp_qdisc_ets_destroy, 1303 .get_stats = mlxsw_sp_qdisc_get_prio_stats, 1304 .clean_stats = mlxsw_sp_setup_tc_qdisc_prio_clean_stats, 1305 .find_class = mlxsw_sp_qdisc_prio_find_class, 1306 .num_classes = IEEE_8021QAZ_MAX_TCS, 1307 }; 1308 1309 /* Linux allows linking of Qdiscs to arbitrary classes (so long as the resulting 1310 * graph is free of cycles). These operations do not change the parent handle 1311 * though, which means it can be incomplete (if there is more than one class 1312 * where the Qdisc in question is grafted) or outright wrong (if the Qdisc was 1313 * linked to a different class and then removed from the original class). 1314 * 1315 * E.g. consider this sequence of operations: 1316 * 1317 * # tc qdisc add dev swp1 root handle 1: prio 1318 * # tc qdisc add dev swp1 parent 1:3 handle 13: red limit 1000000 avpkt 10000 1319 * RED: set bandwidth to 10Mbit 1320 * # tc qdisc link dev swp1 handle 13: parent 1:2 1321 * 1322 * At this point, both 1:2 and 1:3 have the same RED Qdisc instance as their 1323 * child. But RED will still only claim that 1:3 is its parent. If it's removed 1324 * from that band, its only parent will be 1:2, but it will continue to claim 1325 * that it is in fact 1:3. 1326 * 1327 * The notification for child Qdisc replace (e.g. TC_RED_REPLACE) comes before 1328 * the notification for parent graft (e.g. TC_PRIO_GRAFT). We take the replace 1329 * notification to offload the child Qdisc, based on its parent handle, and use 1330 * the graft operation to validate that the class where the child is actually 1331 * grafted corresponds to the parent handle. If the two don't match, we 1332 * unoffload the child. 1333 */ 1334 static int 1335 __mlxsw_sp_qdisc_ets_graft(struct mlxsw_sp_port *mlxsw_sp_port, 1336 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1337 u8 band, u32 child_handle) 1338 { 1339 struct mlxsw_sp_qdisc *old_qdisc; 1340 u32 parent; 1341 1342 if (band < mlxsw_sp_qdisc->num_classes && 1343 mlxsw_sp_qdisc->qdiscs[band].handle == child_handle) 1344 return 0; 1345 1346 if (!child_handle) { 1347 /* This is an invisible FIFO replacing the original Qdisc. 1348 * Ignore it--the original Qdisc's destroy will follow. 1349 */ 1350 return 0; 1351 } 1352 1353 /* See if the grafted qdisc is already offloaded on any tclass. If so, 1354 * unoffload it. 1355 */ 1356 old_qdisc = mlxsw_sp_qdisc_find_by_handle(mlxsw_sp_port, 1357 child_handle); 1358 if (old_qdisc) 1359 mlxsw_sp_qdisc_destroy(mlxsw_sp_port, old_qdisc); 1360 1361 parent = TC_H_MAKE(mlxsw_sp_qdisc->handle, band + 1); 1362 mlxsw_sp_qdisc = mlxsw_sp_qdisc->ops->find_class(mlxsw_sp_qdisc, 1363 parent); 1364 if (!WARN_ON(!mlxsw_sp_qdisc)) 1365 mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 1366 1367 return -EOPNOTSUPP; 1368 } 1369 1370 static int 1371 mlxsw_sp_qdisc_prio_graft(struct mlxsw_sp_port *mlxsw_sp_port, 1372 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, 1373 struct tc_prio_qopt_offload_graft_params *p) 1374 { 1375 return __mlxsw_sp_qdisc_ets_graft(mlxsw_sp_port, mlxsw_sp_qdisc, 1376 p->band, p->child_handle); 1377 } 1378 1379 static int __mlxsw_sp_setup_tc_prio(struct mlxsw_sp_port *mlxsw_sp_port, 1380 struct tc_prio_qopt_offload *p) 1381 { 1382 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc; 1383 1384 mlxsw_sp_qdisc = mlxsw_sp_qdisc_find(mlxsw_sp_port, p->parent, true); 1385 if (!mlxsw_sp_qdisc) 1386 return -EOPNOTSUPP; 1387 1388 if (p->command == TC_PRIO_REPLACE) 1389 return mlxsw_sp_qdisc_replace(mlxsw_sp_port, p->handle, 1390 mlxsw_sp_qdisc, 1391 &mlxsw_sp_qdisc_ops_prio, 1392 &p->replace_params); 1393 1394 if (!mlxsw_sp_qdisc_compare(mlxsw_sp_qdisc, p->handle)) 1395 return -EOPNOTSUPP; 1396 1397 switch (p->command) { 1398 case TC_PRIO_DESTROY: 1399 return mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 1400 case TC_PRIO_STATS: 1401 return mlxsw_sp_qdisc_get_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 1402 &p->stats); 1403 case TC_PRIO_GRAFT: 1404 return mlxsw_sp_qdisc_prio_graft(mlxsw_sp_port, mlxsw_sp_qdisc, 1405 &p->graft_params); 1406 default: 1407 return -EOPNOTSUPP; 1408 } 1409 } 1410 1411 int mlxsw_sp_setup_tc_prio(struct mlxsw_sp_port *mlxsw_sp_port, 1412 struct tc_prio_qopt_offload *p) 1413 { 1414 int err; 1415 1416 mutex_lock(&mlxsw_sp_port->qdisc->lock); 1417 err = __mlxsw_sp_setup_tc_prio(mlxsw_sp_port, p); 1418 mutex_unlock(&mlxsw_sp_port->qdisc->lock); 1419 1420 return err; 1421 } 1422 1423 static int __mlxsw_sp_setup_tc_ets(struct mlxsw_sp_port *mlxsw_sp_port, 1424 struct tc_ets_qopt_offload *p) 1425 { 1426 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc; 1427 1428 mlxsw_sp_qdisc = mlxsw_sp_qdisc_find(mlxsw_sp_port, p->parent, true); 1429 if (!mlxsw_sp_qdisc) 1430 return -EOPNOTSUPP; 1431 1432 if (p->command == TC_ETS_REPLACE) 1433 return mlxsw_sp_qdisc_replace(mlxsw_sp_port, p->handle, 1434 mlxsw_sp_qdisc, 1435 &mlxsw_sp_qdisc_ops_ets, 1436 &p->replace_params); 1437 1438 if (!mlxsw_sp_qdisc_compare(mlxsw_sp_qdisc, p->handle)) 1439 return -EOPNOTSUPP; 1440 1441 switch (p->command) { 1442 case TC_ETS_DESTROY: 1443 return mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); 1444 case TC_ETS_STATS: 1445 return mlxsw_sp_qdisc_get_stats(mlxsw_sp_port, mlxsw_sp_qdisc, 1446 &p->stats); 1447 case TC_ETS_GRAFT: 1448 return __mlxsw_sp_qdisc_ets_graft(mlxsw_sp_port, mlxsw_sp_qdisc, 1449 p->graft_params.band, 1450 p->graft_params.child_handle); 1451 default: 1452 return -EOPNOTSUPP; 1453 } 1454 } 1455 1456 int mlxsw_sp_setup_tc_ets(struct mlxsw_sp_port *mlxsw_sp_port, 1457 struct tc_ets_qopt_offload *p) 1458 { 1459 int err; 1460 1461 mutex_lock(&mlxsw_sp_port->qdisc->lock); 1462 err = __mlxsw_sp_setup_tc_ets(mlxsw_sp_port, p); 1463 mutex_unlock(&mlxsw_sp_port->qdisc->lock); 1464 1465 return err; 1466 } 1467 1468 struct mlxsw_sp_qevent_block { 1469 struct list_head binding_list; 1470 struct list_head mall_entry_list; 1471 struct mlxsw_sp *mlxsw_sp; 1472 }; 1473 1474 struct mlxsw_sp_qevent_binding { 1475 struct list_head list; 1476 struct mlxsw_sp_port *mlxsw_sp_port; 1477 u32 handle; 1478 int tclass_num; 1479 enum mlxsw_sp_span_trigger span_trigger; 1480 unsigned int action_mask; 1481 }; 1482 1483 static LIST_HEAD(mlxsw_sp_qevent_block_cb_list); 1484 1485 static int mlxsw_sp_qevent_span_configure(struct mlxsw_sp *mlxsw_sp, 1486 struct mlxsw_sp_mall_entry *mall_entry, 1487 struct mlxsw_sp_qevent_binding *qevent_binding, 1488 const struct mlxsw_sp_span_agent_parms *agent_parms, 1489 int *p_span_id) 1490 { 1491 enum mlxsw_sp_span_trigger span_trigger = qevent_binding->span_trigger; 1492 struct mlxsw_sp_port *mlxsw_sp_port = qevent_binding->mlxsw_sp_port; 1493 struct mlxsw_sp_span_trigger_parms trigger_parms = {}; 1494 bool ingress; 1495 int span_id; 1496 int err; 1497 1498 err = mlxsw_sp_span_agent_get(mlxsw_sp, &span_id, agent_parms); 1499 if (err) 1500 return err; 1501 1502 ingress = mlxsw_sp_span_trigger_is_ingress(span_trigger); 1503 err = mlxsw_sp_span_analyzed_port_get(mlxsw_sp_port, ingress); 1504 if (err) 1505 goto err_analyzed_port_get; 1506 1507 trigger_parms.span_id = span_id; 1508 trigger_parms.probability_rate = 1; 1509 err = mlxsw_sp_span_agent_bind(mlxsw_sp, span_trigger, mlxsw_sp_port, 1510 &trigger_parms); 1511 if (err) 1512 goto err_agent_bind; 1513 1514 err = mlxsw_sp_span_trigger_enable(mlxsw_sp_port, span_trigger, 1515 qevent_binding->tclass_num); 1516 if (err) 1517 goto err_trigger_enable; 1518 1519 *p_span_id = span_id; 1520 return 0; 1521 1522 err_trigger_enable: 1523 mlxsw_sp_span_agent_unbind(mlxsw_sp, span_trigger, mlxsw_sp_port, 1524 &trigger_parms); 1525 err_agent_bind: 1526 mlxsw_sp_span_analyzed_port_put(mlxsw_sp_port, ingress); 1527 err_analyzed_port_get: 1528 mlxsw_sp_span_agent_put(mlxsw_sp, span_id); 1529 return err; 1530 } 1531 1532 static void mlxsw_sp_qevent_span_deconfigure(struct mlxsw_sp *mlxsw_sp, 1533 struct mlxsw_sp_qevent_binding *qevent_binding, 1534 int span_id) 1535 { 1536 enum mlxsw_sp_span_trigger span_trigger = qevent_binding->span_trigger; 1537 struct mlxsw_sp_port *mlxsw_sp_port = qevent_binding->mlxsw_sp_port; 1538 struct mlxsw_sp_span_trigger_parms trigger_parms = { 1539 .span_id = span_id, 1540 }; 1541 bool ingress; 1542 1543 ingress = mlxsw_sp_span_trigger_is_ingress(span_trigger); 1544 1545 mlxsw_sp_span_trigger_disable(mlxsw_sp_port, span_trigger, 1546 qevent_binding->tclass_num); 1547 mlxsw_sp_span_agent_unbind(mlxsw_sp, span_trigger, mlxsw_sp_port, 1548 &trigger_parms); 1549 mlxsw_sp_span_analyzed_port_put(mlxsw_sp_port, ingress); 1550 mlxsw_sp_span_agent_put(mlxsw_sp, span_id); 1551 } 1552 1553 static int mlxsw_sp_qevent_mirror_configure(struct mlxsw_sp *mlxsw_sp, 1554 struct mlxsw_sp_mall_entry *mall_entry, 1555 struct mlxsw_sp_qevent_binding *qevent_binding) 1556 { 1557 struct mlxsw_sp_span_agent_parms agent_parms = { 1558 .to_dev = mall_entry->mirror.to_dev, 1559 }; 1560 1561 return mlxsw_sp_qevent_span_configure(mlxsw_sp, mall_entry, qevent_binding, 1562 &agent_parms, &mall_entry->mirror.span_id); 1563 } 1564 1565 static void mlxsw_sp_qevent_mirror_deconfigure(struct mlxsw_sp *mlxsw_sp, 1566 struct mlxsw_sp_mall_entry *mall_entry, 1567 struct mlxsw_sp_qevent_binding *qevent_binding) 1568 { 1569 mlxsw_sp_qevent_span_deconfigure(mlxsw_sp, qevent_binding, mall_entry->mirror.span_id); 1570 } 1571 1572 static int mlxsw_sp_qevent_trap_configure(struct mlxsw_sp *mlxsw_sp, 1573 struct mlxsw_sp_mall_entry *mall_entry, 1574 struct mlxsw_sp_qevent_binding *qevent_binding) 1575 { 1576 struct mlxsw_sp_span_agent_parms agent_parms = { 1577 .session_id = MLXSW_SP_SPAN_SESSION_ID_BUFFER, 1578 }; 1579 int err; 1580 1581 err = mlxsw_sp_trap_group_policer_hw_id_get(mlxsw_sp, 1582 DEVLINK_TRAP_GROUP_GENERIC_ID_BUFFER_DROPS, 1583 &agent_parms.policer_enable, 1584 &agent_parms.policer_id); 1585 if (err) 1586 return err; 1587 1588 return mlxsw_sp_qevent_span_configure(mlxsw_sp, mall_entry, qevent_binding, 1589 &agent_parms, &mall_entry->trap.span_id); 1590 } 1591 1592 static void mlxsw_sp_qevent_trap_deconfigure(struct mlxsw_sp *mlxsw_sp, 1593 struct mlxsw_sp_mall_entry *mall_entry, 1594 struct mlxsw_sp_qevent_binding *qevent_binding) 1595 { 1596 mlxsw_sp_qevent_span_deconfigure(mlxsw_sp, qevent_binding, mall_entry->trap.span_id); 1597 } 1598 1599 static int 1600 mlxsw_sp_qevent_entry_configure(struct mlxsw_sp *mlxsw_sp, 1601 struct mlxsw_sp_mall_entry *mall_entry, 1602 struct mlxsw_sp_qevent_binding *qevent_binding, 1603 struct netlink_ext_ack *extack) 1604 { 1605 if (!(BIT(mall_entry->type) & qevent_binding->action_mask)) { 1606 NL_SET_ERR_MSG(extack, "Action not supported at this qevent"); 1607 return -EOPNOTSUPP; 1608 } 1609 1610 switch (mall_entry->type) { 1611 case MLXSW_SP_MALL_ACTION_TYPE_MIRROR: 1612 return mlxsw_sp_qevent_mirror_configure(mlxsw_sp, mall_entry, qevent_binding); 1613 case MLXSW_SP_MALL_ACTION_TYPE_TRAP: 1614 return mlxsw_sp_qevent_trap_configure(mlxsw_sp, mall_entry, qevent_binding); 1615 default: 1616 /* This should have been validated away. */ 1617 WARN_ON(1); 1618 return -EOPNOTSUPP; 1619 } 1620 } 1621 1622 static void mlxsw_sp_qevent_entry_deconfigure(struct mlxsw_sp *mlxsw_sp, 1623 struct mlxsw_sp_mall_entry *mall_entry, 1624 struct mlxsw_sp_qevent_binding *qevent_binding) 1625 { 1626 switch (mall_entry->type) { 1627 case MLXSW_SP_MALL_ACTION_TYPE_MIRROR: 1628 return mlxsw_sp_qevent_mirror_deconfigure(mlxsw_sp, mall_entry, qevent_binding); 1629 case MLXSW_SP_MALL_ACTION_TYPE_TRAP: 1630 return mlxsw_sp_qevent_trap_deconfigure(mlxsw_sp, mall_entry, qevent_binding); 1631 default: 1632 WARN_ON(1); 1633 return; 1634 } 1635 } 1636 1637 static int 1638 mlxsw_sp_qevent_binding_configure(struct mlxsw_sp_qevent_block *qevent_block, 1639 struct mlxsw_sp_qevent_binding *qevent_binding, 1640 struct netlink_ext_ack *extack) 1641 { 1642 struct mlxsw_sp_mall_entry *mall_entry; 1643 int err; 1644 1645 list_for_each_entry(mall_entry, &qevent_block->mall_entry_list, list) { 1646 err = mlxsw_sp_qevent_entry_configure(qevent_block->mlxsw_sp, mall_entry, 1647 qevent_binding, extack); 1648 if (err) 1649 goto err_entry_configure; 1650 } 1651 1652 return 0; 1653 1654 err_entry_configure: 1655 list_for_each_entry_continue_reverse(mall_entry, &qevent_block->mall_entry_list, list) 1656 mlxsw_sp_qevent_entry_deconfigure(qevent_block->mlxsw_sp, mall_entry, 1657 qevent_binding); 1658 return err; 1659 } 1660 1661 static void mlxsw_sp_qevent_binding_deconfigure(struct mlxsw_sp_qevent_block *qevent_block, 1662 struct mlxsw_sp_qevent_binding *qevent_binding) 1663 { 1664 struct mlxsw_sp_mall_entry *mall_entry; 1665 1666 list_for_each_entry(mall_entry, &qevent_block->mall_entry_list, list) 1667 mlxsw_sp_qevent_entry_deconfigure(qevent_block->mlxsw_sp, mall_entry, 1668 qevent_binding); 1669 } 1670 1671 static int 1672 mlxsw_sp_qevent_block_configure(struct mlxsw_sp_qevent_block *qevent_block, 1673 struct netlink_ext_ack *extack) 1674 { 1675 struct mlxsw_sp_qevent_binding *qevent_binding; 1676 int err; 1677 1678 list_for_each_entry(qevent_binding, &qevent_block->binding_list, list) { 1679 err = mlxsw_sp_qevent_binding_configure(qevent_block, 1680 qevent_binding, 1681 extack); 1682 if (err) 1683 goto err_binding_configure; 1684 } 1685 1686 return 0; 1687 1688 err_binding_configure: 1689 list_for_each_entry_continue_reverse(qevent_binding, &qevent_block->binding_list, list) 1690 mlxsw_sp_qevent_binding_deconfigure(qevent_block, qevent_binding); 1691 return err; 1692 } 1693 1694 static void mlxsw_sp_qevent_block_deconfigure(struct mlxsw_sp_qevent_block *qevent_block) 1695 { 1696 struct mlxsw_sp_qevent_binding *qevent_binding; 1697 1698 list_for_each_entry(qevent_binding, &qevent_block->binding_list, list) 1699 mlxsw_sp_qevent_binding_deconfigure(qevent_block, qevent_binding); 1700 } 1701 1702 static struct mlxsw_sp_mall_entry * 1703 mlxsw_sp_qevent_mall_entry_find(struct mlxsw_sp_qevent_block *block, unsigned long cookie) 1704 { 1705 struct mlxsw_sp_mall_entry *mall_entry; 1706 1707 list_for_each_entry(mall_entry, &block->mall_entry_list, list) 1708 if (mall_entry->cookie == cookie) 1709 return mall_entry; 1710 1711 return NULL; 1712 } 1713 1714 static int mlxsw_sp_qevent_mall_replace(struct mlxsw_sp *mlxsw_sp, 1715 struct mlxsw_sp_qevent_block *qevent_block, 1716 struct tc_cls_matchall_offload *f) 1717 { 1718 struct mlxsw_sp_mall_entry *mall_entry; 1719 struct flow_action_entry *act; 1720 int err; 1721 1722 /* It should not currently be possible to replace a matchall rule. So 1723 * this must be a new rule. 1724 */ 1725 if (!list_empty(&qevent_block->mall_entry_list)) { 1726 NL_SET_ERR_MSG(f->common.extack, "At most one filter supported"); 1727 return -EOPNOTSUPP; 1728 } 1729 if (f->rule->action.num_entries != 1) { 1730 NL_SET_ERR_MSG(f->common.extack, "Only singular actions supported"); 1731 return -EOPNOTSUPP; 1732 } 1733 if (f->common.chain_index) { 1734 NL_SET_ERR_MSG(f->common.extack, "Only chain 0 is supported"); 1735 return -EOPNOTSUPP; 1736 } 1737 if (f->common.protocol != htons(ETH_P_ALL)) { 1738 NL_SET_ERR_MSG(f->common.extack, "Protocol matching not supported"); 1739 return -EOPNOTSUPP; 1740 } 1741 1742 act = &f->rule->action.entries[0]; 1743 if (!(act->hw_stats & FLOW_ACTION_HW_STATS_DISABLED)) { 1744 NL_SET_ERR_MSG(f->common.extack, "HW counters not supported on qevents"); 1745 return -EOPNOTSUPP; 1746 } 1747 1748 mall_entry = kzalloc(sizeof(*mall_entry), GFP_KERNEL); 1749 if (!mall_entry) 1750 return -ENOMEM; 1751 mall_entry->cookie = f->cookie; 1752 1753 if (act->id == FLOW_ACTION_MIRRED) { 1754 mall_entry->type = MLXSW_SP_MALL_ACTION_TYPE_MIRROR; 1755 mall_entry->mirror.to_dev = act->dev; 1756 } else if (act->id == FLOW_ACTION_TRAP) { 1757 mall_entry->type = MLXSW_SP_MALL_ACTION_TYPE_TRAP; 1758 } else { 1759 NL_SET_ERR_MSG(f->common.extack, "Unsupported action"); 1760 err = -EOPNOTSUPP; 1761 goto err_unsupported_action; 1762 } 1763 1764 list_add_tail(&mall_entry->list, &qevent_block->mall_entry_list); 1765 1766 err = mlxsw_sp_qevent_block_configure(qevent_block, f->common.extack); 1767 if (err) 1768 goto err_block_configure; 1769 1770 return 0; 1771 1772 err_block_configure: 1773 list_del(&mall_entry->list); 1774 err_unsupported_action: 1775 kfree(mall_entry); 1776 return err; 1777 } 1778 1779 static void mlxsw_sp_qevent_mall_destroy(struct mlxsw_sp_qevent_block *qevent_block, 1780 struct tc_cls_matchall_offload *f) 1781 { 1782 struct mlxsw_sp_mall_entry *mall_entry; 1783 1784 mall_entry = mlxsw_sp_qevent_mall_entry_find(qevent_block, f->cookie); 1785 if (!mall_entry) 1786 return; 1787 1788 mlxsw_sp_qevent_block_deconfigure(qevent_block); 1789 1790 list_del(&mall_entry->list); 1791 kfree(mall_entry); 1792 } 1793 1794 static int mlxsw_sp_qevent_block_mall_cb(struct mlxsw_sp_qevent_block *qevent_block, 1795 struct tc_cls_matchall_offload *f) 1796 { 1797 struct mlxsw_sp *mlxsw_sp = qevent_block->mlxsw_sp; 1798 1799 switch (f->command) { 1800 case TC_CLSMATCHALL_REPLACE: 1801 return mlxsw_sp_qevent_mall_replace(mlxsw_sp, qevent_block, f); 1802 case TC_CLSMATCHALL_DESTROY: 1803 mlxsw_sp_qevent_mall_destroy(qevent_block, f); 1804 return 0; 1805 default: 1806 return -EOPNOTSUPP; 1807 } 1808 } 1809 1810 static int mlxsw_sp_qevent_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 1811 { 1812 struct mlxsw_sp_qevent_block *qevent_block = cb_priv; 1813 1814 switch (type) { 1815 case TC_SETUP_CLSMATCHALL: 1816 return mlxsw_sp_qevent_block_mall_cb(qevent_block, type_data); 1817 default: 1818 return -EOPNOTSUPP; 1819 } 1820 } 1821 1822 static struct mlxsw_sp_qevent_block *mlxsw_sp_qevent_block_create(struct mlxsw_sp *mlxsw_sp, 1823 struct net *net) 1824 { 1825 struct mlxsw_sp_qevent_block *qevent_block; 1826 1827 qevent_block = kzalloc(sizeof(*qevent_block), GFP_KERNEL); 1828 if (!qevent_block) 1829 return NULL; 1830 1831 INIT_LIST_HEAD(&qevent_block->binding_list); 1832 INIT_LIST_HEAD(&qevent_block->mall_entry_list); 1833 qevent_block->mlxsw_sp = mlxsw_sp; 1834 return qevent_block; 1835 } 1836 1837 static void 1838 mlxsw_sp_qevent_block_destroy(struct mlxsw_sp_qevent_block *qevent_block) 1839 { 1840 WARN_ON(!list_empty(&qevent_block->binding_list)); 1841 WARN_ON(!list_empty(&qevent_block->mall_entry_list)); 1842 kfree(qevent_block); 1843 } 1844 1845 static void mlxsw_sp_qevent_block_release(void *cb_priv) 1846 { 1847 struct mlxsw_sp_qevent_block *qevent_block = cb_priv; 1848 1849 mlxsw_sp_qevent_block_destroy(qevent_block); 1850 } 1851 1852 static struct mlxsw_sp_qevent_binding * 1853 mlxsw_sp_qevent_binding_create(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle, int tclass_num, 1854 enum mlxsw_sp_span_trigger span_trigger, 1855 unsigned int action_mask) 1856 { 1857 struct mlxsw_sp_qevent_binding *binding; 1858 1859 binding = kzalloc(sizeof(*binding), GFP_KERNEL); 1860 if (!binding) 1861 return ERR_PTR(-ENOMEM); 1862 1863 binding->mlxsw_sp_port = mlxsw_sp_port; 1864 binding->handle = handle; 1865 binding->tclass_num = tclass_num; 1866 binding->span_trigger = span_trigger; 1867 binding->action_mask = action_mask; 1868 return binding; 1869 } 1870 1871 static void 1872 mlxsw_sp_qevent_binding_destroy(struct mlxsw_sp_qevent_binding *binding) 1873 { 1874 kfree(binding); 1875 } 1876 1877 static struct mlxsw_sp_qevent_binding * 1878 mlxsw_sp_qevent_binding_lookup(struct mlxsw_sp_qevent_block *block, 1879 struct mlxsw_sp_port *mlxsw_sp_port, 1880 u32 handle, 1881 enum mlxsw_sp_span_trigger span_trigger) 1882 { 1883 struct mlxsw_sp_qevent_binding *qevent_binding; 1884 1885 list_for_each_entry(qevent_binding, &block->binding_list, list) 1886 if (qevent_binding->mlxsw_sp_port == mlxsw_sp_port && 1887 qevent_binding->handle == handle && 1888 qevent_binding->span_trigger == span_trigger) 1889 return qevent_binding; 1890 return NULL; 1891 } 1892 1893 static int 1894 mlxsw_sp_setup_tc_block_qevent_bind(struct mlxsw_sp_port *mlxsw_sp_port, 1895 struct flow_block_offload *f, 1896 enum mlxsw_sp_span_trigger span_trigger, 1897 unsigned int action_mask) 1898 { 1899 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 1900 struct mlxsw_sp_qevent_binding *qevent_binding; 1901 struct mlxsw_sp_qevent_block *qevent_block; 1902 struct flow_block_cb *block_cb; 1903 struct mlxsw_sp_qdisc *qdisc; 1904 bool register_block = false; 1905 int err; 1906 1907 block_cb = flow_block_cb_lookup(f->block, mlxsw_sp_qevent_block_cb, mlxsw_sp); 1908 if (!block_cb) { 1909 qevent_block = mlxsw_sp_qevent_block_create(mlxsw_sp, f->net); 1910 if (!qevent_block) 1911 return -ENOMEM; 1912 block_cb = flow_block_cb_alloc(mlxsw_sp_qevent_block_cb, mlxsw_sp, qevent_block, 1913 mlxsw_sp_qevent_block_release); 1914 if (IS_ERR(block_cb)) { 1915 mlxsw_sp_qevent_block_destroy(qevent_block); 1916 return PTR_ERR(block_cb); 1917 } 1918 register_block = true; 1919 } else { 1920 qevent_block = flow_block_cb_priv(block_cb); 1921 } 1922 flow_block_cb_incref(block_cb); 1923 1924 qdisc = mlxsw_sp_qdisc_find_by_handle(mlxsw_sp_port, f->sch->handle); 1925 if (!qdisc) { 1926 NL_SET_ERR_MSG(f->extack, "Qdisc not offloaded"); 1927 err = -ENOENT; 1928 goto err_find_qdisc; 1929 } 1930 1931 if (WARN_ON(mlxsw_sp_qevent_binding_lookup(qevent_block, mlxsw_sp_port, f->sch->handle, 1932 span_trigger))) { 1933 err = -EEXIST; 1934 goto err_binding_exists; 1935 } 1936 1937 qevent_binding = mlxsw_sp_qevent_binding_create(mlxsw_sp_port, 1938 f->sch->handle, 1939 qdisc->tclass_num, 1940 span_trigger, 1941 action_mask); 1942 if (IS_ERR(qevent_binding)) { 1943 err = PTR_ERR(qevent_binding); 1944 goto err_binding_create; 1945 } 1946 1947 err = mlxsw_sp_qevent_binding_configure(qevent_block, qevent_binding, 1948 f->extack); 1949 if (err) 1950 goto err_binding_configure; 1951 1952 list_add(&qevent_binding->list, &qevent_block->binding_list); 1953 1954 if (register_block) { 1955 flow_block_cb_add(block_cb, f); 1956 list_add_tail(&block_cb->driver_list, &mlxsw_sp_qevent_block_cb_list); 1957 } 1958 1959 return 0; 1960 1961 err_binding_configure: 1962 mlxsw_sp_qevent_binding_destroy(qevent_binding); 1963 err_binding_create: 1964 err_binding_exists: 1965 err_find_qdisc: 1966 if (!flow_block_cb_decref(block_cb)) 1967 flow_block_cb_free(block_cb); 1968 return err; 1969 } 1970 1971 static void mlxsw_sp_setup_tc_block_qevent_unbind(struct mlxsw_sp_port *mlxsw_sp_port, 1972 struct flow_block_offload *f, 1973 enum mlxsw_sp_span_trigger span_trigger) 1974 { 1975 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 1976 struct mlxsw_sp_qevent_binding *qevent_binding; 1977 struct mlxsw_sp_qevent_block *qevent_block; 1978 struct flow_block_cb *block_cb; 1979 1980 block_cb = flow_block_cb_lookup(f->block, mlxsw_sp_qevent_block_cb, mlxsw_sp); 1981 if (!block_cb) 1982 return; 1983 qevent_block = flow_block_cb_priv(block_cb); 1984 1985 qevent_binding = mlxsw_sp_qevent_binding_lookup(qevent_block, mlxsw_sp_port, f->sch->handle, 1986 span_trigger); 1987 if (!qevent_binding) 1988 return; 1989 1990 list_del(&qevent_binding->list); 1991 mlxsw_sp_qevent_binding_deconfigure(qevent_block, qevent_binding); 1992 mlxsw_sp_qevent_binding_destroy(qevent_binding); 1993 1994 if (!flow_block_cb_decref(block_cb)) { 1995 flow_block_cb_remove(block_cb, f); 1996 list_del(&block_cb->driver_list); 1997 } 1998 } 1999 2000 static int 2001 mlxsw_sp_setup_tc_block_qevent(struct mlxsw_sp_port *mlxsw_sp_port, 2002 struct flow_block_offload *f, 2003 enum mlxsw_sp_span_trigger span_trigger, 2004 unsigned int action_mask) 2005 { 2006 f->driver_block_list = &mlxsw_sp_qevent_block_cb_list; 2007 2008 switch (f->command) { 2009 case FLOW_BLOCK_BIND: 2010 return mlxsw_sp_setup_tc_block_qevent_bind(mlxsw_sp_port, f, 2011 span_trigger, 2012 action_mask); 2013 case FLOW_BLOCK_UNBIND: 2014 mlxsw_sp_setup_tc_block_qevent_unbind(mlxsw_sp_port, f, span_trigger); 2015 return 0; 2016 default: 2017 return -EOPNOTSUPP; 2018 } 2019 } 2020 2021 int mlxsw_sp_setup_tc_block_qevent_early_drop(struct mlxsw_sp_port *mlxsw_sp_port, 2022 struct flow_block_offload *f) 2023 { 2024 unsigned int action_mask = BIT(MLXSW_SP_MALL_ACTION_TYPE_MIRROR) | 2025 BIT(MLXSW_SP_MALL_ACTION_TYPE_TRAP); 2026 2027 return mlxsw_sp_setup_tc_block_qevent(mlxsw_sp_port, f, 2028 MLXSW_SP_SPAN_TRIGGER_EARLY_DROP, 2029 action_mask); 2030 } 2031 2032 int mlxsw_sp_setup_tc_block_qevent_mark(struct mlxsw_sp_port *mlxsw_sp_port, 2033 struct flow_block_offload *f) 2034 { 2035 unsigned int action_mask = BIT(MLXSW_SP_MALL_ACTION_TYPE_MIRROR); 2036 2037 return mlxsw_sp_setup_tc_block_qevent(mlxsw_sp_port, f, 2038 MLXSW_SP_SPAN_TRIGGER_ECN, 2039 action_mask); 2040 } 2041 2042 int mlxsw_sp_tc_qdisc_init(struct mlxsw_sp_port *mlxsw_sp_port) 2043 { 2044 struct mlxsw_sp_qdisc_state *qdisc_state; 2045 2046 qdisc_state = kzalloc(sizeof(*qdisc_state), GFP_KERNEL); 2047 if (!qdisc_state) 2048 return -ENOMEM; 2049 2050 mutex_init(&qdisc_state->lock); 2051 qdisc_state->root_qdisc.prio_bitmap = 0xff; 2052 qdisc_state->root_qdisc.tclass_num = MLXSW_SP_PORT_DEFAULT_TCLASS; 2053 mlxsw_sp_port->qdisc = qdisc_state; 2054 return 0; 2055 } 2056 2057 void mlxsw_sp_tc_qdisc_fini(struct mlxsw_sp_port *mlxsw_sp_port) 2058 { 2059 mutex_destroy(&mlxsw_sp_port->qdisc->lock); 2060 kfree(mlxsw_sp_port->qdisc); 2061 } 2062