1214baf22SMaxim Mikityanskiy // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2214baf22SMaxim Mikityanskiy /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3b6459415SJakub Kicinski #include <net/sch_generic.h> 4214baf22SMaxim Mikityanskiy 5efe31799SSaeed Mahameed #include <net/pkt_cls.h> 6214baf22SMaxim Mikityanskiy #include "en.h" 7214baf22SMaxim Mikityanskiy #include "params.h" 8214baf22SMaxim Mikityanskiy #include "../qos.h" 9462b0059SMoshe Tal #include "en/htb.h" 10214baf22SMaxim Mikityanskiy 11462b0059SMoshe Tal struct qos_sq_callback_params { 1228df4a01SMoshe Tal struct mlx5e_priv *priv; 13462b0059SMoshe Tal struct mlx5e_channels *chs; 14aaffda6bSSaeed Mahameed }; 15aaffda6bSSaeed Mahameed 1680743c4fSTariq Toukan int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes) 1780743c4fSTariq Toukan { 1880743c4fSTariq Toukan if (nbytes < BYTES_IN_MBIT) { 1980743c4fSTariq Toukan qos_warn(mdev, "Input rate (%llu Bytes/sec) below minimum supported (%u Bytes/sec)\n", 2080743c4fSTariq Toukan nbytes, BYTES_IN_MBIT); 2180743c4fSTariq Toukan return -EINVAL; 2280743c4fSTariq Toukan } 2380743c4fSTariq Toukan return 0; 2480743c4fSTariq Toukan } 2580743c4fSTariq Toukan 2680743c4fSTariq Toukan static u32 mlx5e_qos_bytes2mbits(struct mlx5_core_dev *mdev, u64 nbytes) 2780743c4fSTariq Toukan { 2880743c4fSTariq Toukan return div_u64(nbytes, BYTES_IN_MBIT); 2980743c4fSTariq Toukan } 3080743c4fSTariq Toukan 31214baf22SMaxim Mikityanskiy int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev) 32214baf22SMaxim Mikityanskiy { 33214baf22SMaxim Mikityanskiy return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev)); 34214baf22SMaxim Mikityanskiy } 35214baf22SMaxim Mikityanskiy 36214baf22SMaxim Mikityanskiy /* TX datapath API */ 37214baf22SMaxim Mikityanskiy 38462b0059SMoshe Tal u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) 39214baf22SMaxim Mikityanskiy { 40214baf22SMaxim Mikityanskiy /* These channel params are safe to access from the datapath, because: 414f8d1d3aSMoshe Tal * 1. This function is called only after checking selq->htb_maj_id != 0, 42214baf22SMaxim Mikityanskiy * and the number of queues can't change while HTB offload is active. 434f8d1d3aSMoshe Tal * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for 44214baf22SMaxim Mikityanskiy * mlx5e_select_queue to finish while holding priv->state_lock, 45214baf22SMaxim Mikityanskiy * preventing other code from changing the number of queues. 46214baf22SMaxim Mikityanskiy */ 47214baf22SMaxim Mikityanskiy bool is_ptp = MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS); 48214baf22SMaxim Mikityanskiy 4986d747a3STariq Toukan return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid; 50214baf22SMaxim Mikityanskiy } 51214baf22SMaxim Mikityanskiy 5266d95936SMoshe Tal /* SQ lifecycle */ 5366d95936SMoshe Tal 54214baf22SMaxim Mikityanskiy static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) 55214baf22SMaxim Mikityanskiy { 56214baf22SMaxim Mikityanskiy struct mlx5e_params *params = &priv->channels.params; 57214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 58214baf22SMaxim Mikityanskiy struct mlx5e_channel *c; 59214baf22SMaxim Mikityanskiy int ix; 60214baf22SMaxim Mikityanskiy 61214baf22SMaxim Mikityanskiy ix = qid % params->num_channels; 62214baf22SMaxim Mikityanskiy qid /= params->num_channels; 63214baf22SMaxim Mikityanskiy c = priv->channels.c[ix]; 64214baf22SMaxim Mikityanskiy 65214baf22SMaxim Mikityanskiy qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 66214baf22SMaxim Mikityanskiy return mlx5e_state_dereference(priv, qos_sqs[qid]); 67214baf22SMaxim Mikityanskiy } 68214baf22SMaxim Mikityanskiy 69462b0059SMoshe Tal int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, 70462b0059SMoshe Tal u16 node_qid, u32 hw_id) 71214baf22SMaxim Mikityanskiy { 72214baf22SMaxim Mikityanskiy struct mlx5e_create_cq_param ccp = {}; 73214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 74214baf22SMaxim Mikityanskiy struct mlx5e_sq_param param_sq; 75214baf22SMaxim Mikityanskiy struct mlx5e_cq_param param_cq; 76214baf22SMaxim Mikityanskiy int txq_ix, ix, qid, err = 0; 77214baf22SMaxim Mikityanskiy struct mlx5e_params *params; 78214baf22SMaxim Mikityanskiy struct mlx5e_channel *c; 79214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 80214baf22SMaxim Mikityanskiy 81214baf22SMaxim Mikityanskiy params = &chs->params; 82214baf22SMaxim Mikityanskiy 83462b0059SMoshe Tal txq_ix = mlx5e_qid_from_qos(chs, node_qid); 84214baf22SMaxim Mikityanskiy 85462b0059SMoshe Tal WARN_ON(node_qid > priv->htb_max_qos_sqs); 86462b0059SMoshe Tal if (node_qid == priv->htb_max_qos_sqs) { 87214baf22SMaxim Mikityanskiy struct mlx5e_sq_stats *stats, **stats_list = NULL; 88214baf22SMaxim Mikityanskiy 89db83f24dSMoshe Tal if (priv->htb_max_qos_sqs == 0) { 90214baf22SMaxim Mikityanskiy stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev), 91214baf22SMaxim Mikityanskiy sizeof(*stats_list), 92214baf22SMaxim Mikityanskiy GFP_KERNEL); 93214baf22SMaxim Mikityanskiy if (!stats_list) 94214baf22SMaxim Mikityanskiy return -ENOMEM; 95214baf22SMaxim Mikityanskiy } 96214baf22SMaxim Mikityanskiy stats = kzalloc(sizeof(*stats), GFP_KERNEL); 97214baf22SMaxim Mikityanskiy if (!stats) { 98214baf22SMaxim Mikityanskiy kvfree(stats_list); 99214baf22SMaxim Mikityanskiy return -ENOMEM; 100214baf22SMaxim Mikityanskiy } 101214baf22SMaxim Mikityanskiy if (stats_list) 102db83f24dSMoshe Tal WRITE_ONCE(priv->htb_qos_sq_stats, stats_list); 103462b0059SMoshe Tal WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats); 104db83f24dSMoshe Tal /* Order htb_max_qos_sqs increment after writing the array pointer. 105214baf22SMaxim Mikityanskiy * Pairs with smp_load_acquire in en_stats.c. 106214baf22SMaxim Mikityanskiy */ 107db83f24dSMoshe Tal smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1); 108214baf22SMaxim Mikityanskiy } 109214baf22SMaxim Mikityanskiy 110462b0059SMoshe Tal ix = node_qid % params->num_channels; 111462b0059SMoshe Tal qid = node_qid / params->num_channels; 112214baf22SMaxim Mikityanskiy c = chs->c[ix]; 113214baf22SMaxim Mikityanskiy 114214baf22SMaxim Mikityanskiy qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 115214baf22SMaxim Mikityanskiy sq = kzalloc(sizeof(*sq), GFP_KERNEL); 116214baf22SMaxim Mikityanskiy 117214baf22SMaxim Mikityanskiy if (!sq) 118214baf22SMaxim Mikityanskiy return -ENOMEM; 119214baf22SMaxim Mikityanskiy 120214baf22SMaxim Mikityanskiy mlx5e_build_create_cq_param(&ccp, c); 121214baf22SMaxim Mikityanskiy 122214baf22SMaxim Mikityanskiy memset(¶m_sq, 0, sizeof(param_sq)); 123214baf22SMaxim Mikityanskiy memset(¶m_cq, 0, sizeof(param_cq)); 12489564920STariq Toukan mlx5e_build_sq_param(priv->mdev, params, ¶m_sq); 12589564920STariq Toukan mlx5e_build_tx_cq_param(priv->mdev, params, ¶m_cq); 126214baf22SMaxim Mikityanskiy err = mlx5e_open_cq(priv, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); 127214baf22SMaxim Mikityanskiy if (err) 128214baf22SMaxim Mikityanskiy goto err_free_sq; 129214baf22SMaxim Mikityanskiy err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, 130462b0059SMoshe Tal ¶m_sq, sq, 0, hw_id, 131462b0059SMoshe Tal priv->htb_qos_sq_stats[node_qid]); 132214baf22SMaxim Mikityanskiy if (err) 133214baf22SMaxim Mikityanskiy goto err_close_cq; 134214baf22SMaxim Mikityanskiy 135214baf22SMaxim Mikityanskiy rcu_assign_pointer(qos_sqs[qid], sq); 136214baf22SMaxim Mikityanskiy 137214baf22SMaxim Mikityanskiy return 0; 138214baf22SMaxim Mikityanskiy 139214baf22SMaxim Mikityanskiy err_close_cq: 140214baf22SMaxim Mikityanskiy mlx5e_close_cq(&sq->cq); 141214baf22SMaxim Mikityanskiy err_free_sq: 142214baf22SMaxim Mikityanskiy kfree(sq); 143214baf22SMaxim Mikityanskiy return err; 144214baf22SMaxim Mikityanskiy } 145214baf22SMaxim Mikityanskiy 146462b0059SMoshe Tal static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id) 147214baf22SMaxim Mikityanskiy { 148462b0059SMoshe Tal struct qos_sq_callback_params *cb_params = data; 149462b0059SMoshe Tal 150462b0059SMoshe Tal return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id); 151462b0059SMoshe Tal } 152462b0059SMoshe Tal 153462b0059SMoshe Tal int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id) 154462b0059SMoshe Tal { 155462b0059SMoshe Tal struct mlx5e_priv *priv = data; 156214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 15717c84cb4SMaxim Mikityanskiy u16 qid; 158214baf22SMaxim Mikityanskiy 159462b0059SMoshe Tal sq = mlx5e_get_qos_sq(priv, node_qid); 160214baf22SMaxim Mikityanskiy 161462b0059SMoshe Tal qid = mlx5e_qid_from_qos(&priv->channels, node_qid); 16217c84cb4SMaxim Mikityanskiy 16317c84cb4SMaxim Mikityanskiy /* If it's a new queue, it will be marked as started at this point. 16417c84cb4SMaxim Mikityanskiy * Stop it before updating txq2sq. 16517c84cb4SMaxim Mikityanskiy */ 16617c84cb4SMaxim Mikityanskiy mlx5e_tx_disable_queue(netdev_get_tx_queue(priv->netdev, qid)); 16717c84cb4SMaxim Mikityanskiy 16817c84cb4SMaxim Mikityanskiy priv->txq2sq[qid] = sq; 169214baf22SMaxim Mikityanskiy 170214baf22SMaxim Mikityanskiy /* Make the change to txq2sq visible before the queue is started. 171214baf22SMaxim Mikityanskiy * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 172214baf22SMaxim Mikityanskiy * which pairs with this barrier. 173214baf22SMaxim Mikityanskiy */ 174214baf22SMaxim Mikityanskiy smp_wmb(); 175214baf22SMaxim Mikityanskiy 176462b0059SMoshe Tal qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid); 177214baf22SMaxim Mikityanskiy mlx5e_activate_txqsq(sq); 178462b0059SMoshe Tal 179462b0059SMoshe Tal return 0; 180214baf22SMaxim Mikityanskiy } 181214baf22SMaxim Mikityanskiy 182462b0059SMoshe Tal void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) 183214baf22SMaxim Mikityanskiy { 184214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 185214baf22SMaxim Mikityanskiy 186214baf22SMaxim Mikityanskiy sq = mlx5e_get_qos_sq(priv, qid); 187214baf22SMaxim Mikityanskiy if (!sq) /* Handle the case when the SQ failed to open. */ 188214baf22SMaxim Mikityanskiy return; 189214baf22SMaxim Mikityanskiy 190214baf22SMaxim Mikityanskiy qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid); 191214baf22SMaxim Mikityanskiy mlx5e_deactivate_txqsq(sq); 192214baf22SMaxim Mikityanskiy 193214baf22SMaxim Mikityanskiy priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL; 19417c84cb4SMaxim Mikityanskiy 19517c84cb4SMaxim Mikityanskiy /* Make the change to txq2sq visible before the queue is started again. 19617c84cb4SMaxim Mikityanskiy * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 19717c84cb4SMaxim Mikityanskiy * which pairs with this barrier. 19817c84cb4SMaxim Mikityanskiy */ 19917c84cb4SMaxim Mikityanskiy smp_wmb(); 200214baf22SMaxim Mikityanskiy } 201214baf22SMaxim Mikityanskiy 202462b0059SMoshe Tal void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) 203214baf22SMaxim Mikityanskiy { 204214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 205214baf22SMaxim Mikityanskiy struct mlx5e_params *params; 206214baf22SMaxim Mikityanskiy struct mlx5e_channel *c; 207214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 208214baf22SMaxim Mikityanskiy int ix; 209214baf22SMaxim Mikityanskiy 210214baf22SMaxim Mikityanskiy params = &priv->channels.params; 211214baf22SMaxim Mikityanskiy 212214baf22SMaxim Mikityanskiy ix = qid % params->num_channels; 213214baf22SMaxim Mikityanskiy qid /= params->num_channels; 214214baf22SMaxim Mikityanskiy c = priv->channels.c[ix]; 215214baf22SMaxim Mikityanskiy qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 216214baf22SMaxim Mikityanskiy sq = rcu_replace_pointer(qos_sqs[qid], NULL, lockdep_is_held(&priv->state_lock)); 217214baf22SMaxim Mikityanskiy if (!sq) /* Handle the case when the SQ failed to open. */ 218214baf22SMaxim Mikityanskiy return; 219214baf22SMaxim Mikityanskiy 220214baf22SMaxim Mikityanskiy synchronize_rcu(); /* Sync with NAPI. */ 221214baf22SMaxim Mikityanskiy 222214baf22SMaxim Mikityanskiy mlx5e_close_txqsq(sq); 223214baf22SMaxim Mikityanskiy mlx5e_close_cq(&sq->cq); 224214baf22SMaxim Mikityanskiy kfree(sq); 225214baf22SMaxim Mikityanskiy } 226214baf22SMaxim Mikityanskiy 227214baf22SMaxim Mikityanskiy void mlx5e_qos_close_queues(struct mlx5e_channel *c) 228214baf22SMaxim Mikityanskiy { 229214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 230214baf22SMaxim Mikityanskiy int i; 231214baf22SMaxim Mikityanskiy 232214baf22SMaxim Mikityanskiy qos_sqs = rcu_replace_pointer(c->qos_sqs, NULL, lockdep_is_held(&c->priv->state_lock)); 233214baf22SMaxim Mikityanskiy if (!qos_sqs) 234214baf22SMaxim Mikityanskiy return; 235214baf22SMaxim Mikityanskiy synchronize_rcu(); /* Sync with NAPI. */ 236214baf22SMaxim Mikityanskiy 237214baf22SMaxim Mikityanskiy for (i = 0; i < c->qos_sqs_size; i++) { 238214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 239214baf22SMaxim Mikityanskiy 240214baf22SMaxim Mikityanskiy sq = mlx5e_state_dereference(c->priv, qos_sqs[i]); 241214baf22SMaxim Mikityanskiy if (!sq) /* Handle the case when the SQ failed to open. */ 242214baf22SMaxim Mikityanskiy continue; 243214baf22SMaxim Mikityanskiy 244214baf22SMaxim Mikityanskiy mlx5e_close_txqsq(sq); 245214baf22SMaxim Mikityanskiy mlx5e_close_cq(&sq->cq); 246214baf22SMaxim Mikityanskiy kfree(sq); 247214baf22SMaxim Mikityanskiy } 248214baf22SMaxim Mikityanskiy 249214baf22SMaxim Mikityanskiy kvfree(qos_sqs); 250214baf22SMaxim Mikityanskiy } 251214baf22SMaxim Mikityanskiy 252462b0059SMoshe Tal void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) 253214baf22SMaxim Mikityanskiy { 254214baf22SMaxim Mikityanskiy int i; 255214baf22SMaxim Mikityanskiy 256214baf22SMaxim Mikityanskiy for (i = 0; i < chs->num; i++) 257214baf22SMaxim Mikityanskiy mlx5e_qos_close_queues(chs->c[i]); 258214baf22SMaxim Mikityanskiy } 259214baf22SMaxim Mikityanskiy 260462b0059SMoshe Tal int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 261214baf22SMaxim Mikityanskiy { 262214baf22SMaxim Mikityanskiy u16 qos_sqs_size; 263214baf22SMaxim Mikityanskiy int i; 264214baf22SMaxim Mikityanskiy 265214baf22SMaxim Mikityanskiy qos_sqs_size = DIV_ROUND_UP(mlx5e_qos_max_leaf_nodes(priv->mdev), chs->num); 266214baf22SMaxim Mikityanskiy 267214baf22SMaxim Mikityanskiy for (i = 0; i < chs->num; i++) { 268214baf22SMaxim Mikityanskiy struct mlx5e_txqsq **sqs; 269214baf22SMaxim Mikityanskiy 270214baf22SMaxim Mikityanskiy sqs = kvcalloc(qos_sqs_size, sizeof(struct mlx5e_txqsq *), GFP_KERNEL); 271214baf22SMaxim Mikityanskiy if (!sqs) 272214baf22SMaxim Mikityanskiy goto err_free; 273214baf22SMaxim Mikityanskiy 274214baf22SMaxim Mikityanskiy WRITE_ONCE(chs->c[i]->qos_sqs_size, qos_sqs_size); 275214baf22SMaxim Mikityanskiy smp_wmb(); /* Pairs with mlx5e_napi_poll. */ 276214baf22SMaxim Mikityanskiy rcu_assign_pointer(chs->c[i]->qos_sqs, sqs); 277214baf22SMaxim Mikityanskiy } 278214baf22SMaxim Mikityanskiy 279214baf22SMaxim Mikityanskiy return 0; 280214baf22SMaxim Mikityanskiy 281214baf22SMaxim Mikityanskiy err_free: 282214baf22SMaxim Mikityanskiy while (--i >= 0) { 283214baf22SMaxim Mikityanskiy struct mlx5e_txqsq **sqs; 284214baf22SMaxim Mikityanskiy 285214baf22SMaxim Mikityanskiy sqs = rcu_replace_pointer(chs->c[i]->qos_sqs, NULL, 286214baf22SMaxim Mikityanskiy lockdep_is_held(&priv->state_lock)); 287214baf22SMaxim Mikityanskiy 288214baf22SMaxim Mikityanskiy synchronize_rcu(); /* Sync with NAPI. */ 289214baf22SMaxim Mikityanskiy kvfree(sqs); 290214baf22SMaxim Mikityanskiy } 291214baf22SMaxim Mikityanskiy return -ENOMEM; 292214baf22SMaxim Mikityanskiy } 293214baf22SMaxim Mikityanskiy 294214baf22SMaxim Mikityanskiy int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 295214baf22SMaxim Mikityanskiy { 296462b0059SMoshe Tal struct qos_sq_callback_params callback_params; 297462b0059SMoshe Tal int err; 298214baf22SMaxim Mikityanskiy 299214baf22SMaxim Mikityanskiy err = mlx5e_qos_alloc_queues(priv, chs); 300214baf22SMaxim Mikityanskiy if (err) 301214baf22SMaxim Mikityanskiy return err; 302214baf22SMaxim Mikityanskiy 303462b0059SMoshe Tal callback_params.priv = priv; 304462b0059SMoshe Tal callback_params.chs = chs; 305462b0059SMoshe Tal 306462b0059SMoshe Tal err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params); 307214baf22SMaxim Mikityanskiy if (err) { 308214baf22SMaxim Mikityanskiy mlx5e_qos_close_all_queues(chs); 309214baf22SMaxim Mikityanskiy return err; 310214baf22SMaxim Mikityanskiy } 311214baf22SMaxim Mikityanskiy 312214baf22SMaxim Mikityanskiy return 0; 313214baf22SMaxim Mikityanskiy } 314214baf22SMaxim Mikityanskiy 315214baf22SMaxim Mikityanskiy void mlx5e_qos_activate_queues(struct mlx5e_priv *priv) 316214baf22SMaxim Mikityanskiy { 317462b0059SMoshe Tal mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv); 318214baf22SMaxim Mikityanskiy } 319214baf22SMaxim Mikityanskiy 320214baf22SMaxim Mikityanskiy void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) 321214baf22SMaxim Mikityanskiy { 322214baf22SMaxim Mikityanskiy struct mlx5e_params *params = &c->priv->channels.params; 323214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 324214baf22SMaxim Mikityanskiy int i; 325214baf22SMaxim Mikityanskiy 326214baf22SMaxim Mikityanskiy qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs); 327214baf22SMaxim Mikityanskiy if (!qos_sqs) 328214baf22SMaxim Mikityanskiy return; 329214baf22SMaxim Mikityanskiy 330214baf22SMaxim Mikityanskiy for (i = 0; i < c->qos_sqs_size; i++) { 331214baf22SMaxim Mikityanskiy u16 qid = params->num_channels * i + c->ix; 332214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 333214baf22SMaxim Mikityanskiy 334214baf22SMaxim Mikityanskiy sq = mlx5e_state_dereference(c->priv, qos_sqs[i]); 335214baf22SMaxim Mikityanskiy if (!sq) /* Handle the case when the SQ failed to open. */ 336214baf22SMaxim Mikityanskiy continue; 337214baf22SMaxim Mikityanskiy 338214baf22SMaxim Mikityanskiy qos_dbg(c->mdev, "Deactivate QoS SQ qid %u\n", qid); 339214baf22SMaxim Mikityanskiy mlx5e_deactivate_txqsq(sq); 340214baf22SMaxim Mikityanskiy 341214baf22SMaxim Mikityanskiy /* The queue is disabled, no synchronization with datapath is needed. */ 342214baf22SMaxim Mikityanskiy c->priv->txq2sq[mlx5e_qid_from_qos(&c->priv->channels, qid)] = NULL; 343214baf22SMaxim Mikityanskiy } 344214baf22SMaxim Mikityanskiy } 345214baf22SMaxim Mikityanskiy 346462b0059SMoshe Tal void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) 347214baf22SMaxim Mikityanskiy { 348214baf22SMaxim Mikityanskiy int i; 349214baf22SMaxim Mikityanskiy 350214baf22SMaxim Mikityanskiy for (i = 0; i < chs->num; i++) 351214baf22SMaxim Mikityanskiy mlx5e_qos_deactivate_queues(chs->c[i]); 352214baf22SMaxim Mikityanskiy } 353214baf22SMaxim Mikityanskiy 354462b0059SMoshe Tal void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) 355214baf22SMaxim Mikityanskiy { 356214baf22SMaxim Mikityanskiy qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid); 357214baf22SMaxim Mikityanskiy netdev_tx_reset_queue(txq); 358214baf22SMaxim Mikityanskiy netif_tx_start_queue(txq); 359214baf22SMaxim Mikityanskiy } 360214baf22SMaxim Mikityanskiy 361462b0059SMoshe Tal void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) 362214baf22SMaxim Mikityanskiy { 363214baf22SMaxim Mikityanskiy struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid); 364214baf22SMaxim Mikityanskiy struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 365214baf22SMaxim Mikityanskiy 366214baf22SMaxim Mikityanskiy if (!qdisc) 367214baf22SMaxim Mikityanskiy return; 368214baf22SMaxim Mikityanskiy 369214baf22SMaxim Mikityanskiy spin_lock_bh(qdisc_lock(qdisc)); 370214baf22SMaxim Mikityanskiy qdisc_reset(qdisc); 371214baf22SMaxim Mikityanskiy spin_unlock_bh(qdisc_lock(qdisc)); 372214baf22SMaxim Mikityanskiy } 373214baf22SMaxim Mikityanskiy 37428df4a01SMoshe Tal int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt) 375efe31799SSaeed Mahameed { 37628df4a01SMoshe Tal struct mlx5e_htb *htb = priv->htb; 377efe31799SSaeed Mahameed int res; 378efe31799SSaeed Mahameed 37928df4a01SMoshe Tal if (!htb && htb_qopt->command != TC_HTB_CREATE) 380aaffda6bSSaeed Mahameed return -EINVAL; 381aaffda6bSSaeed Mahameed 382*12e7789aSNaveen Mamindlapalli if (htb_qopt->prio) { 383*12e7789aSNaveen Mamindlapalli NL_SET_ERR_MSG_MOD(htb_qopt->extack, 384*12e7789aSNaveen Mamindlapalli "prio parameter is not supported by device with HTB offload enabled."); 385*12e7789aSNaveen Mamindlapalli return -EOPNOTSUPP; 386*12e7789aSNaveen Mamindlapalli } 387*12e7789aSNaveen Mamindlapalli 38828df4a01SMoshe Tal switch (htb_qopt->command) { 389efe31799SSaeed Mahameed case TC_HTB_CREATE: 390aaffda6bSSaeed Mahameed if (!mlx5_qos_is_supported(priv->mdev)) { 39128df4a01SMoshe Tal NL_SET_ERR_MSG_MOD(htb_qopt->extack, 392aaffda6bSSaeed Mahameed "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); 393aaffda6bSSaeed Mahameed return -EOPNOTSUPP; 394aaffda6bSSaeed Mahameed } 395aaffda6bSSaeed Mahameed priv->htb = mlx5e_htb_alloc(); 39628df4a01SMoshe Tal htb = priv->htb; 39728df4a01SMoshe Tal if (!htb) 398aaffda6bSSaeed Mahameed return -ENOMEM; 39928df4a01SMoshe Tal res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv); 400aaffda6bSSaeed Mahameed if (res) { 40128df4a01SMoshe Tal mlx5e_htb_free(htb); 402aaffda6bSSaeed Mahameed priv->htb = NULL; 403aaffda6bSSaeed Mahameed } 404aaffda6bSSaeed Mahameed return res; 405efe31799SSaeed Mahameed case TC_HTB_DESTROY: 40628df4a01SMoshe Tal mlx5e_htb_cleanup(htb); 40728df4a01SMoshe Tal mlx5e_htb_free(htb); 408aaffda6bSSaeed Mahameed priv->htb = NULL; 409aaffda6bSSaeed Mahameed return 0; 410efe31799SSaeed Mahameed case TC_HTB_LEAF_ALLOC_QUEUE: 41128df4a01SMoshe Tal res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid, 41228df4a01SMoshe Tal htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); 413efe31799SSaeed Mahameed if (res < 0) 414efe31799SSaeed Mahameed return res; 41528df4a01SMoshe Tal htb_qopt->qid = res; 416efe31799SSaeed Mahameed return 0; 417efe31799SSaeed Mahameed case TC_HTB_LEAF_TO_INNER: 41828df4a01SMoshe Tal return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid, 41928df4a01SMoshe Tal htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); 420efe31799SSaeed Mahameed case TC_HTB_LEAF_DEL: 42128df4a01SMoshe Tal return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack); 422efe31799SSaeed Mahameed case TC_HTB_LEAF_DEL_LAST: 423efe31799SSaeed Mahameed case TC_HTB_LEAF_DEL_LAST_FORCE: 42428df4a01SMoshe Tal return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid, 42528df4a01SMoshe Tal htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE, 42628df4a01SMoshe Tal htb_qopt->extack); 427efe31799SSaeed Mahameed case TC_HTB_NODE_MODIFY: 42828df4a01SMoshe Tal return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil, 42928df4a01SMoshe Tal htb_qopt->extack); 430efe31799SSaeed Mahameed case TC_HTB_LEAF_QUERY_QUEUE: 4313685eed5SMoshe Tal res = mlx5e_htb_get_txq_by_classid(htb, htb_qopt->classid); 432efe31799SSaeed Mahameed if (res < 0) 433efe31799SSaeed Mahameed return res; 43428df4a01SMoshe Tal htb_qopt->qid = res; 435efe31799SSaeed Mahameed return 0; 436efe31799SSaeed Mahameed default: 437efe31799SSaeed Mahameed return -EOPNOTSUPP; 438efe31799SSaeed Mahameed } 439efe31799SSaeed Mahameed } 440efe31799SSaeed Mahameed 44180743c4fSTariq Toukan struct mlx5e_mqprio_rl { 44280743c4fSTariq Toukan struct mlx5_core_dev *mdev; 44380743c4fSTariq Toukan u32 root_id; 44480743c4fSTariq Toukan u32 *leaves_id; 44580743c4fSTariq Toukan u8 num_tc; 44680743c4fSTariq Toukan }; 44780743c4fSTariq Toukan 44880743c4fSTariq Toukan struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_alloc(void) 44980743c4fSTariq Toukan { 45080743c4fSTariq Toukan return kvzalloc(sizeof(struct mlx5e_mqprio_rl), GFP_KERNEL); 45180743c4fSTariq Toukan } 45280743c4fSTariq Toukan 45380743c4fSTariq Toukan void mlx5e_mqprio_rl_free(struct mlx5e_mqprio_rl *rl) 45480743c4fSTariq Toukan { 45580743c4fSTariq Toukan kvfree(rl); 45680743c4fSTariq Toukan } 45780743c4fSTariq Toukan 45880743c4fSTariq Toukan int mlx5e_mqprio_rl_init(struct mlx5e_mqprio_rl *rl, struct mlx5_core_dev *mdev, u8 num_tc, 45980743c4fSTariq Toukan u64 max_rate[]) 46080743c4fSTariq Toukan { 46180743c4fSTariq Toukan int err; 46280743c4fSTariq Toukan int tc; 46380743c4fSTariq Toukan 46480743c4fSTariq Toukan if (!mlx5_qos_is_supported(mdev)) { 46580743c4fSTariq Toukan qos_warn(mdev, "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); 46680743c4fSTariq Toukan return -EOPNOTSUPP; 46780743c4fSTariq Toukan } 46880743c4fSTariq Toukan if (num_tc > mlx5e_qos_max_leaf_nodes(mdev)) 46980743c4fSTariq Toukan return -EINVAL; 47080743c4fSTariq Toukan 47180743c4fSTariq Toukan rl->mdev = mdev; 47280743c4fSTariq Toukan rl->num_tc = num_tc; 47380743c4fSTariq Toukan rl->leaves_id = kvcalloc(num_tc, sizeof(*rl->leaves_id), GFP_KERNEL); 47480743c4fSTariq Toukan if (!rl->leaves_id) 47580743c4fSTariq Toukan return -ENOMEM; 47680743c4fSTariq Toukan 47780743c4fSTariq Toukan err = mlx5_qos_create_root_node(mdev, &rl->root_id); 47880743c4fSTariq Toukan if (err) 47980743c4fSTariq Toukan goto err_free_leaves; 48080743c4fSTariq Toukan 48180743c4fSTariq Toukan qos_dbg(mdev, "Root created, id %#x\n", rl->root_id); 48280743c4fSTariq Toukan 48380743c4fSTariq Toukan for (tc = 0; tc < num_tc; tc++) { 48480743c4fSTariq Toukan u32 max_average_bw; 48580743c4fSTariq Toukan 48680743c4fSTariq Toukan max_average_bw = mlx5e_qos_bytes2mbits(mdev, max_rate[tc]); 48780743c4fSTariq Toukan err = mlx5_qos_create_leaf_node(mdev, rl->root_id, 0, max_average_bw, 48880743c4fSTariq Toukan &rl->leaves_id[tc]); 48980743c4fSTariq Toukan if (err) 49080743c4fSTariq Toukan goto err_destroy_leaves; 49180743c4fSTariq Toukan 49280743c4fSTariq Toukan qos_dbg(mdev, "Leaf[%d] created, id %#x, max average bw %u Mbits/sec\n", 49380743c4fSTariq Toukan tc, rl->leaves_id[tc], max_average_bw); 49480743c4fSTariq Toukan } 49580743c4fSTariq Toukan return 0; 49680743c4fSTariq Toukan 49780743c4fSTariq Toukan err_destroy_leaves: 49880743c4fSTariq Toukan while (--tc >= 0) 49980743c4fSTariq Toukan mlx5_qos_destroy_node(mdev, rl->leaves_id[tc]); 50080743c4fSTariq Toukan mlx5_qos_destroy_node(mdev, rl->root_id); 50180743c4fSTariq Toukan err_free_leaves: 50280743c4fSTariq Toukan kvfree(rl->leaves_id); 50380743c4fSTariq Toukan return err; 50480743c4fSTariq Toukan } 50580743c4fSTariq Toukan 50680743c4fSTariq Toukan void mlx5e_mqprio_rl_cleanup(struct mlx5e_mqprio_rl *rl) 50780743c4fSTariq Toukan { 50880743c4fSTariq Toukan int tc; 50980743c4fSTariq Toukan 51080743c4fSTariq Toukan for (tc = 0; tc < rl->num_tc; tc++) 51180743c4fSTariq Toukan mlx5_qos_destroy_node(rl->mdev, rl->leaves_id[tc]); 51280743c4fSTariq Toukan mlx5_qos_destroy_node(rl->mdev, rl->root_id); 51380743c4fSTariq Toukan kvfree(rl->leaves_id); 51480743c4fSTariq Toukan } 51580743c4fSTariq Toukan 51680743c4fSTariq Toukan int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_id) 51780743c4fSTariq Toukan { 51880743c4fSTariq Toukan if (tc >= rl->num_tc) 51980743c4fSTariq Toukan return -EINVAL; 52080743c4fSTariq Toukan 52180743c4fSTariq Toukan *hw_id = rl->leaves_id[tc]; 52280743c4fSTariq Toukan return 0; 52380743c4fSTariq Toukan } 524