1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3 #include <net/sch_generic.h> 4 5 #include <net/pkt_cls.h> 6 #include "en.h" 7 #include "params.h" 8 #include "../qos.h" 9 #include "en/htb.h" 10 11 struct qos_sq_callback_params { 12 struct mlx5e_priv *priv; 13 struct mlx5e_channels *chs; 14 }; 15 16 int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes) 17 { 18 if (nbytes < BYTES_IN_MBIT) { 19 qos_warn(mdev, "Input rate (%llu Bytes/sec) below minimum supported (%u Bytes/sec)\n", 20 nbytes, BYTES_IN_MBIT); 21 return -EINVAL; 22 } 23 return 0; 24 } 25 26 static u32 mlx5e_qos_bytes2mbits(struct mlx5_core_dev *mdev, u64 nbytes) 27 { 28 return div_u64(nbytes, BYTES_IN_MBIT); 29 } 30 31 int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev) 32 { 33 return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev)); 34 } 35 36 /* TX datapath API */ 37 38 u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) 39 { 40 /* These channel params are safe to access from the datapath, because: 41 * 1. This function is called only after checking selq->htb_maj_id != 0, 42 * and the number of queues can't change while HTB offload is active. 43 * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for 44 * mlx5e_select_queue to finish while holding priv->state_lock, 45 * preventing other code from changing the number of queues. 46 */ 47 bool is_ptp = MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS); 48 49 return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid; 50 } 51 52 /* SQ lifecycle */ 53 54 static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) 55 { 56 struct mlx5e_params *params = &priv->channels.params; 57 struct mlx5e_txqsq __rcu **qos_sqs; 58 struct mlx5e_channel *c; 59 int ix; 60 61 ix = qid % params->num_channels; 62 qid /= params->num_channels; 63 c = priv->channels.c[ix]; 64 65 qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 66 return mlx5e_state_dereference(priv, qos_sqs[qid]); 67 } 68 69 int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, 70 u16 node_qid, u32 hw_id) 71 { 72 struct mlx5e_create_cq_param ccp = {}; 73 struct mlx5e_txqsq __rcu **qos_sqs; 74 struct mlx5e_sq_param param_sq; 75 struct mlx5e_cq_param param_cq; 76 int txq_ix, ix, qid, err = 0; 77 struct mlx5e_params *params; 78 struct mlx5e_channel *c; 79 struct mlx5e_txqsq *sq; 80 81 params = &chs->params; 82 83 txq_ix = mlx5e_qid_from_qos(chs, node_qid); 84 85 WARN_ON(node_qid > priv->htb_max_qos_sqs); 86 if (node_qid == priv->htb_max_qos_sqs) { 87 struct mlx5e_sq_stats *stats, **stats_list = NULL; 88 89 if (priv->htb_max_qos_sqs == 0) { 90 stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev), 91 sizeof(*stats_list), 92 GFP_KERNEL); 93 if (!stats_list) 94 return -ENOMEM; 95 } 96 stats = kzalloc(sizeof(*stats), GFP_KERNEL); 97 if (!stats) { 98 kvfree(stats_list); 99 return -ENOMEM; 100 } 101 if (stats_list) 102 WRITE_ONCE(priv->htb_qos_sq_stats, stats_list); 103 WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats); 104 /* Order htb_max_qos_sqs increment after writing the array pointer. 105 * Pairs with smp_load_acquire in en_stats.c. 106 */ 107 smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1); 108 } 109 110 ix = node_qid % params->num_channels; 111 qid = node_qid / params->num_channels; 112 c = chs->c[ix]; 113 114 qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 115 sq = kzalloc(sizeof(*sq), GFP_KERNEL); 116 117 if (!sq) 118 return -ENOMEM; 119 120 mlx5e_build_create_cq_param(&ccp, c); 121 122 memset(¶m_sq, 0, sizeof(param_sq)); 123 memset(¶m_cq, 0, sizeof(param_cq)); 124 mlx5e_build_sq_param(priv->mdev, params, ¶m_sq); 125 mlx5e_build_tx_cq_param(priv->mdev, params, ¶m_cq); 126 err = mlx5e_open_cq(priv, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); 127 if (err) 128 goto err_free_sq; 129 err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, 130 ¶m_sq, sq, 0, hw_id, 131 priv->htb_qos_sq_stats[node_qid]); 132 if (err) 133 goto err_close_cq; 134 135 rcu_assign_pointer(qos_sqs[qid], sq); 136 137 return 0; 138 139 err_close_cq: 140 mlx5e_close_cq(&sq->cq); 141 err_free_sq: 142 kfree(sq); 143 return err; 144 } 145 146 static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id) 147 { 148 struct qos_sq_callback_params *cb_params = data; 149 150 return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id); 151 } 152 153 int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id) 154 { 155 struct mlx5e_priv *priv = data; 156 struct mlx5e_txqsq *sq; 157 u16 qid; 158 159 sq = mlx5e_get_qos_sq(priv, node_qid); 160 161 qid = mlx5e_qid_from_qos(&priv->channels, node_qid); 162 163 /* If it's a new queue, it will be marked as started at this point. 164 * Stop it before updating txq2sq. 165 */ 166 mlx5e_tx_disable_queue(netdev_get_tx_queue(priv->netdev, qid)); 167 168 priv->txq2sq[qid] = sq; 169 170 /* Make the change to txq2sq visible before the queue is started. 171 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 172 * which pairs with this barrier. 173 */ 174 smp_wmb(); 175 176 qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid); 177 mlx5e_activate_txqsq(sq); 178 179 return 0; 180 } 181 182 void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) 183 { 184 struct mlx5e_txqsq *sq; 185 186 sq = mlx5e_get_qos_sq(priv, qid); 187 if (!sq) /* Handle the case when the SQ failed to open. */ 188 return; 189 190 qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid); 191 mlx5e_deactivate_txqsq(sq); 192 193 priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL; 194 195 /* Make the change to txq2sq visible before the queue is started again. 196 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 197 * which pairs with this barrier. 198 */ 199 smp_wmb(); 200 } 201 202 void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) 203 { 204 struct mlx5e_txqsq __rcu **qos_sqs; 205 struct mlx5e_params *params; 206 struct mlx5e_channel *c; 207 struct mlx5e_txqsq *sq; 208 int ix; 209 210 params = &priv->channels.params; 211 212 ix = qid % params->num_channels; 213 qid /= params->num_channels; 214 c = priv->channels.c[ix]; 215 qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 216 sq = rcu_replace_pointer(qos_sqs[qid], NULL, lockdep_is_held(&priv->state_lock)); 217 if (!sq) /* Handle the case when the SQ failed to open. */ 218 return; 219 220 synchronize_rcu(); /* Sync with NAPI. */ 221 222 mlx5e_close_txqsq(sq); 223 mlx5e_close_cq(&sq->cq); 224 kfree(sq); 225 } 226 227 void mlx5e_qos_close_queues(struct mlx5e_channel *c) 228 { 229 struct mlx5e_txqsq __rcu **qos_sqs; 230 int i; 231 232 qos_sqs = rcu_replace_pointer(c->qos_sqs, NULL, lockdep_is_held(&c->priv->state_lock)); 233 if (!qos_sqs) 234 return; 235 synchronize_rcu(); /* Sync with NAPI. */ 236 237 for (i = 0; i < c->qos_sqs_size; i++) { 238 struct mlx5e_txqsq *sq; 239 240 sq = mlx5e_state_dereference(c->priv, qos_sqs[i]); 241 if (!sq) /* Handle the case when the SQ failed to open. */ 242 continue; 243 244 mlx5e_close_txqsq(sq); 245 mlx5e_close_cq(&sq->cq); 246 kfree(sq); 247 } 248 249 kvfree(qos_sqs); 250 } 251 252 void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) 253 { 254 int i; 255 256 for (i = 0; i < chs->num; i++) 257 mlx5e_qos_close_queues(chs->c[i]); 258 } 259 260 int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 261 { 262 u16 qos_sqs_size; 263 int i; 264 265 qos_sqs_size = DIV_ROUND_UP(mlx5e_qos_max_leaf_nodes(priv->mdev), chs->num); 266 267 for (i = 0; i < chs->num; i++) { 268 struct mlx5e_txqsq **sqs; 269 270 sqs = kvcalloc(qos_sqs_size, sizeof(struct mlx5e_txqsq *), GFP_KERNEL); 271 if (!sqs) 272 goto err_free; 273 274 WRITE_ONCE(chs->c[i]->qos_sqs_size, qos_sqs_size); 275 smp_wmb(); /* Pairs with mlx5e_napi_poll. */ 276 rcu_assign_pointer(chs->c[i]->qos_sqs, sqs); 277 } 278 279 return 0; 280 281 err_free: 282 while (--i >= 0) { 283 struct mlx5e_txqsq **sqs; 284 285 sqs = rcu_replace_pointer(chs->c[i]->qos_sqs, NULL, 286 lockdep_is_held(&priv->state_lock)); 287 288 synchronize_rcu(); /* Sync with NAPI. */ 289 kvfree(sqs); 290 } 291 return -ENOMEM; 292 } 293 294 int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 295 { 296 struct qos_sq_callback_params callback_params; 297 int err; 298 299 err = mlx5e_qos_alloc_queues(priv, chs); 300 if (err) 301 return err; 302 303 callback_params.priv = priv; 304 callback_params.chs = chs; 305 306 err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params); 307 if (err) { 308 mlx5e_qos_close_all_queues(chs); 309 return err; 310 } 311 312 return 0; 313 } 314 315 void mlx5e_qos_activate_queues(struct mlx5e_priv *priv) 316 { 317 mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv); 318 } 319 320 void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) 321 { 322 struct mlx5e_params *params = &c->priv->channels.params; 323 struct mlx5e_txqsq __rcu **qos_sqs; 324 int i; 325 326 qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs); 327 if (!qos_sqs) 328 return; 329 330 for (i = 0; i < c->qos_sqs_size; i++) { 331 u16 qid = params->num_channels * i + c->ix; 332 struct mlx5e_txqsq *sq; 333 334 sq = mlx5e_state_dereference(c->priv, qos_sqs[i]); 335 if (!sq) /* Handle the case when the SQ failed to open. */ 336 continue; 337 338 qos_dbg(c->mdev, "Deactivate QoS SQ qid %u\n", qid); 339 mlx5e_deactivate_txqsq(sq); 340 341 /* The queue is disabled, no synchronization with datapath is needed. */ 342 c->priv->txq2sq[mlx5e_qid_from_qos(&c->priv->channels, qid)] = NULL; 343 } 344 } 345 346 void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) 347 { 348 int i; 349 350 for (i = 0; i < chs->num; i++) 351 mlx5e_qos_deactivate_queues(chs->c[i]); 352 } 353 354 void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) 355 { 356 qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid); 357 netdev_tx_reset_queue(txq); 358 netif_tx_start_queue(txq); 359 } 360 361 void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) 362 { 363 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid); 364 struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 365 366 if (!qdisc) 367 return; 368 369 spin_lock_bh(qdisc_lock(qdisc)); 370 qdisc_reset(qdisc); 371 spin_unlock_bh(qdisc_lock(qdisc)); 372 } 373 374 int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt) 375 { 376 struct mlx5e_htb *htb = priv->htb; 377 int res; 378 379 if (!htb && htb_qopt->command != TC_HTB_CREATE) 380 return -EINVAL; 381 382 switch (htb_qopt->command) { 383 case TC_HTB_CREATE: 384 if (!mlx5_qos_is_supported(priv->mdev)) { 385 NL_SET_ERR_MSG_MOD(htb_qopt->extack, 386 "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); 387 return -EOPNOTSUPP; 388 } 389 priv->htb = mlx5e_htb_alloc(); 390 htb = priv->htb; 391 if (!htb) 392 return -ENOMEM; 393 res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv); 394 if (res) { 395 mlx5e_htb_free(htb); 396 priv->htb = NULL; 397 } 398 return res; 399 case TC_HTB_DESTROY: 400 mlx5e_htb_cleanup(htb); 401 mlx5e_htb_free(htb); 402 priv->htb = NULL; 403 return 0; 404 case TC_HTB_LEAF_ALLOC_QUEUE: 405 res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid, 406 htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); 407 if (res < 0) 408 return res; 409 htb_qopt->qid = res; 410 return 0; 411 case TC_HTB_LEAF_TO_INNER: 412 return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid, 413 htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); 414 case TC_HTB_LEAF_DEL: 415 return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack); 416 case TC_HTB_LEAF_DEL_LAST: 417 case TC_HTB_LEAF_DEL_LAST_FORCE: 418 return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid, 419 htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE, 420 htb_qopt->extack); 421 case TC_HTB_NODE_MODIFY: 422 return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil, 423 htb_qopt->extack); 424 case TC_HTB_LEAF_QUERY_QUEUE: 425 res = mlx5e_htb_get_txq_by_classid(htb, htb_qopt->classid); 426 if (res < 0) 427 return res; 428 htb_qopt->qid = res; 429 return 0; 430 default: 431 return -EOPNOTSUPP; 432 } 433 } 434 435 struct mlx5e_mqprio_rl { 436 struct mlx5_core_dev *mdev; 437 u32 root_id; 438 u32 *leaves_id; 439 u8 num_tc; 440 }; 441 442 struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_alloc(void) 443 { 444 return kvzalloc(sizeof(struct mlx5e_mqprio_rl), GFP_KERNEL); 445 } 446 447 void mlx5e_mqprio_rl_free(struct mlx5e_mqprio_rl *rl) 448 { 449 kvfree(rl); 450 } 451 452 int mlx5e_mqprio_rl_init(struct mlx5e_mqprio_rl *rl, struct mlx5_core_dev *mdev, u8 num_tc, 453 u64 max_rate[]) 454 { 455 int err; 456 int tc; 457 458 if (!mlx5_qos_is_supported(mdev)) { 459 qos_warn(mdev, "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); 460 return -EOPNOTSUPP; 461 } 462 if (num_tc > mlx5e_qos_max_leaf_nodes(mdev)) 463 return -EINVAL; 464 465 rl->mdev = mdev; 466 rl->num_tc = num_tc; 467 rl->leaves_id = kvcalloc(num_tc, sizeof(*rl->leaves_id), GFP_KERNEL); 468 if (!rl->leaves_id) 469 return -ENOMEM; 470 471 err = mlx5_qos_create_root_node(mdev, &rl->root_id); 472 if (err) 473 goto err_free_leaves; 474 475 qos_dbg(mdev, "Root created, id %#x\n", rl->root_id); 476 477 for (tc = 0; tc < num_tc; tc++) { 478 u32 max_average_bw; 479 480 max_average_bw = mlx5e_qos_bytes2mbits(mdev, max_rate[tc]); 481 err = mlx5_qos_create_leaf_node(mdev, rl->root_id, 0, max_average_bw, 482 &rl->leaves_id[tc]); 483 if (err) 484 goto err_destroy_leaves; 485 486 qos_dbg(mdev, "Leaf[%d] created, id %#x, max average bw %u Mbits/sec\n", 487 tc, rl->leaves_id[tc], max_average_bw); 488 } 489 return 0; 490 491 err_destroy_leaves: 492 while (--tc >= 0) 493 mlx5_qos_destroy_node(mdev, rl->leaves_id[tc]); 494 mlx5_qos_destroy_node(mdev, rl->root_id); 495 err_free_leaves: 496 kvfree(rl->leaves_id); 497 return err; 498 } 499 500 void mlx5e_mqprio_rl_cleanup(struct mlx5e_mqprio_rl *rl) 501 { 502 int tc; 503 504 for (tc = 0; tc < rl->num_tc; tc++) 505 mlx5_qos_destroy_node(rl->mdev, rl->leaves_id[tc]); 506 mlx5_qos_destroy_node(rl->mdev, rl->root_id); 507 kvfree(rl->leaves_id); 508 } 509 510 int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_id) 511 { 512 if (tc >= rl->num_tc) 513 return -EINVAL; 514 515 *hw_id = rl->leaves_id[tc]; 516 return 0; 517 } 518 519