1214baf22SMaxim Mikityanskiy // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2214baf22SMaxim Mikityanskiy /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3b6459415SJakub Kicinski #include <net/sch_generic.h> 4214baf22SMaxim Mikityanskiy 5efe31799SSaeed Mahameed #include <net/pkt_cls.h> 6214baf22SMaxim Mikityanskiy #include "en.h" 7214baf22SMaxim Mikityanskiy #include "params.h" 8214baf22SMaxim Mikityanskiy #include "../qos.h" 9214baf22SMaxim Mikityanskiy 10214baf22SMaxim Mikityanskiy #define BYTES_IN_MBIT 125000 11214baf22SMaxim Mikityanskiy 12aaffda6bSSaeed Mahameed struct mlx5e_htb { 13aaffda6bSSaeed Mahameed DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES)); 14aaffda6bSSaeed Mahameed DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES); 15*28df4a01SMoshe Tal struct mlx5_core_dev *mdev; 16*28df4a01SMoshe Tal struct net_device *netdev; 17*28df4a01SMoshe Tal struct mlx5e_priv *priv; 18*28df4a01SMoshe Tal struct mlx5e_selq *selq; 19aaffda6bSSaeed Mahameed }; 20aaffda6bSSaeed Mahameed 2180743c4fSTariq Toukan int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes) 2280743c4fSTariq Toukan { 2380743c4fSTariq Toukan if (nbytes < BYTES_IN_MBIT) { 2480743c4fSTariq Toukan qos_warn(mdev, "Input rate (%llu Bytes/sec) below minimum supported (%u Bytes/sec)\n", 2580743c4fSTariq Toukan nbytes, BYTES_IN_MBIT); 2680743c4fSTariq Toukan return -EINVAL; 2780743c4fSTariq Toukan } 2880743c4fSTariq Toukan return 0; 2980743c4fSTariq Toukan } 3080743c4fSTariq Toukan 3180743c4fSTariq Toukan static u32 mlx5e_qos_bytes2mbits(struct mlx5_core_dev *mdev, u64 nbytes) 3280743c4fSTariq Toukan { 3380743c4fSTariq Toukan return div_u64(nbytes, BYTES_IN_MBIT); 3480743c4fSTariq Toukan } 3580743c4fSTariq Toukan 36214baf22SMaxim Mikityanskiy int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev) 37214baf22SMaxim Mikityanskiy { 38214baf22SMaxim Mikityanskiy return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev)); 39214baf22SMaxim Mikityanskiy } 40214baf22SMaxim Mikityanskiy 41*28df4a01SMoshe Tal int mlx5e_qos_cur_leaf_nodes(struct mlx5e_htb *htb) 42214baf22SMaxim Mikityanskiy { 43aaffda6bSSaeed Mahameed int last; 44214baf22SMaxim Mikityanskiy 45*28df4a01SMoshe Tal last = find_last_bit(htb->qos_used_qids, mlx5e_qos_max_leaf_nodes(htb->mdev)); 46*28df4a01SMoshe Tal return last == mlx5e_qos_max_leaf_nodes(htb->mdev) ? 0 : last + 1; 47214baf22SMaxim Mikityanskiy } 48214baf22SMaxim Mikityanskiy 49214baf22SMaxim Mikityanskiy /* Software representation of the QoS tree (internal to this file) */ 50214baf22SMaxim Mikityanskiy 51*28df4a01SMoshe Tal static int mlx5e_find_unused_qos_qid(struct mlx5e_htb *htb) 52214baf22SMaxim Mikityanskiy { 53*28df4a01SMoshe Tal int size = mlx5e_qos_max_leaf_nodes(htb->mdev); 54*28df4a01SMoshe Tal struct mlx5e_priv *priv = htb->priv; 55214baf22SMaxim Mikityanskiy int res; 56214baf22SMaxim Mikityanskiy 57214baf22SMaxim Mikityanskiy WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__); 58*28df4a01SMoshe Tal res = find_first_zero_bit(htb->qos_used_qids, size); 59214baf22SMaxim Mikityanskiy 60214baf22SMaxim Mikityanskiy return res == size ? -ENOSPC : res; 61214baf22SMaxim Mikityanskiy } 62214baf22SMaxim Mikityanskiy 63214baf22SMaxim Mikityanskiy struct mlx5e_qos_node { 64214baf22SMaxim Mikityanskiy struct hlist_node hnode; 65214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *parent; 66214baf22SMaxim Mikityanskiy u64 rate; 67214baf22SMaxim Mikityanskiy u32 bw_share; 68214baf22SMaxim Mikityanskiy u32 max_average_bw; 69214baf22SMaxim Mikityanskiy u32 hw_id; 70214baf22SMaxim Mikityanskiy u32 classid; /* 16-bit, except root. */ 71214baf22SMaxim Mikityanskiy u16 qid; 72214baf22SMaxim Mikityanskiy }; 73214baf22SMaxim Mikityanskiy 74214baf22SMaxim Mikityanskiy #define MLX5E_QOS_QID_INNER 0xffff 75214baf22SMaxim Mikityanskiy #define MLX5E_HTB_CLASSID_ROOT 0xffffffff 76214baf22SMaxim Mikityanskiy 77214baf22SMaxim Mikityanskiy static struct mlx5e_qos_node * 78*28df4a01SMoshe Tal mlx5e_sw_node_create_leaf(struct mlx5e_htb *htb, u16 classid, u16 qid, 79214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *parent) 80214baf22SMaxim Mikityanskiy { 81214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node; 82214baf22SMaxim Mikityanskiy 83214baf22SMaxim Mikityanskiy node = kzalloc(sizeof(*node), GFP_KERNEL); 84214baf22SMaxim Mikityanskiy if (!node) 85214baf22SMaxim Mikityanskiy return ERR_PTR(-ENOMEM); 86214baf22SMaxim Mikityanskiy 87214baf22SMaxim Mikityanskiy node->parent = parent; 88214baf22SMaxim Mikityanskiy 89214baf22SMaxim Mikityanskiy node->qid = qid; 90*28df4a01SMoshe Tal __set_bit(qid, htb->qos_used_qids); 91214baf22SMaxim Mikityanskiy 92214baf22SMaxim Mikityanskiy node->classid = classid; 93*28df4a01SMoshe Tal hash_add_rcu(htb->qos_tc2node, &node->hnode, classid); 94214baf22SMaxim Mikityanskiy 95*28df4a01SMoshe Tal mlx5e_update_tx_netdev_queues(htb->priv); 96214baf22SMaxim Mikityanskiy 97214baf22SMaxim Mikityanskiy return node; 98214baf22SMaxim Mikityanskiy } 99214baf22SMaxim Mikityanskiy 100*28df4a01SMoshe Tal static struct mlx5e_qos_node *mlx5e_sw_node_create_root(struct mlx5e_htb *htb) 101214baf22SMaxim Mikityanskiy { 102214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node; 103214baf22SMaxim Mikityanskiy 104214baf22SMaxim Mikityanskiy node = kzalloc(sizeof(*node), GFP_KERNEL); 105214baf22SMaxim Mikityanskiy if (!node) 106214baf22SMaxim Mikityanskiy return ERR_PTR(-ENOMEM); 107214baf22SMaxim Mikityanskiy 108214baf22SMaxim Mikityanskiy node->qid = MLX5E_QOS_QID_INNER; 109214baf22SMaxim Mikityanskiy node->classid = MLX5E_HTB_CLASSID_ROOT; 110*28df4a01SMoshe Tal hash_add_rcu(htb->qos_tc2node, &node->hnode, node->classid); 111214baf22SMaxim Mikityanskiy 112214baf22SMaxim Mikityanskiy return node; 113214baf22SMaxim Mikityanskiy } 114214baf22SMaxim Mikityanskiy 115*28df4a01SMoshe Tal static struct mlx5e_qos_node *mlx5e_sw_node_find(struct mlx5e_htb *htb, u32 classid) 116214baf22SMaxim Mikityanskiy { 117214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node = NULL; 118214baf22SMaxim Mikityanskiy 119*28df4a01SMoshe Tal hash_for_each_possible(htb->qos_tc2node, node, hnode, classid) { 120214baf22SMaxim Mikityanskiy if (node->classid == classid) 121214baf22SMaxim Mikityanskiy break; 122214baf22SMaxim Mikityanskiy } 123214baf22SMaxim Mikityanskiy 124214baf22SMaxim Mikityanskiy return node; 125214baf22SMaxim Mikityanskiy } 126214baf22SMaxim Mikityanskiy 127*28df4a01SMoshe Tal static struct mlx5e_qos_node *mlx5e_sw_node_find_rcu(struct mlx5e_htb *htb, u32 classid) 128214baf22SMaxim Mikityanskiy { 129214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node = NULL; 130214baf22SMaxim Mikityanskiy 131*28df4a01SMoshe Tal hash_for_each_possible_rcu(htb->qos_tc2node, node, hnode, classid) { 132214baf22SMaxim Mikityanskiy if (node->classid == classid) 133214baf22SMaxim Mikityanskiy break; 134214baf22SMaxim Mikityanskiy } 135214baf22SMaxim Mikityanskiy 136214baf22SMaxim Mikityanskiy return node; 137214baf22SMaxim Mikityanskiy } 138214baf22SMaxim Mikityanskiy 139*28df4a01SMoshe Tal static void mlx5e_sw_node_delete(struct mlx5e_htb *htb, struct mlx5e_qos_node *node) 140214baf22SMaxim Mikityanskiy { 141214baf22SMaxim Mikityanskiy hash_del_rcu(&node->hnode); 142214baf22SMaxim Mikityanskiy if (node->qid != MLX5E_QOS_QID_INNER) { 143*28df4a01SMoshe Tal __clear_bit(node->qid, htb->qos_used_qids); 144*28df4a01SMoshe Tal mlx5e_update_tx_netdev_queues(htb->priv); 145214baf22SMaxim Mikityanskiy } 14617c84cb4SMaxim Mikityanskiy /* Make sure this qid is no longer selected by mlx5e_select_queue, so 14717c84cb4SMaxim Mikityanskiy * that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue. 14817c84cb4SMaxim Mikityanskiy */ 14917c84cb4SMaxim Mikityanskiy synchronize_net(); 15017c84cb4SMaxim Mikityanskiy kfree(node); 151214baf22SMaxim Mikityanskiy } 152214baf22SMaxim Mikityanskiy 153214baf22SMaxim Mikityanskiy /* TX datapath API */ 154214baf22SMaxim Mikityanskiy 155214baf22SMaxim Mikityanskiy static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) 156214baf22SMaxim Mikityanskiy { 157214baf22SMaxim Mikityanskiy /* These channel params are safe to access from the datapath, because: 1584f8d1d3aSMoshe Tal * 1. This function is called only after checking selq->htb_maj_id != 0, 159214baf22SMaxim Mikityanskiy * and the number of queues can't change while HTB offload is active. 1604f8d1d3aSMoshe Tal * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for 161214baf22SMaxim Mikityanskiy * mlx5e_select_queue to finish while holding priv->state_lock, 162214baf22SMaxim Mikityanskiy * preventing other code from changing the number of queues. 163214baf22SMaxim Mikityanskiy */ 164214baf22SMaxim Mikityanskiy bool is_ptp = MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS); 165214baf22SMaxim Mikityanskiy 16686d747a3STariq Toukan return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid; 167214baf22SMaxim Mikityanskiy } 168214baf22SMaxim Mikityanskiy 169*28df4a01SMoshe Tal int mlx5e_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid) 170214baf22SMaxim Mikityanskiy { 171214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node; 172214baf22SMaxim Mikityanskiy u16 qid; 173214baf22SMaxim Mikityanskiy int res; 174214baf22SMaxim Mikityanskiy 175214baf22SMaxim Mikityanskiy rcu_read_lock(); 176214baf22SMaxim Mikityanskiy 177*28df4a01SMoshe Tal node = mlx5e_sw_node_find_rcu(htb, classid); 178214baf22SMaxim Mikityanskiy if (!node) { 179214baf22SMaxim Mikityanskiy res = -ENOENT; 180214baf22SMaxim Mikityanskiy goto out; 181214baf22SMaxim Mikityanskiy } 182214baf22SMaxim Mikityanskiy qid = READ_ONCE(node->qid); 183214baf22SMaxim Mikityanskiy if (qid == MLX5E_QOS_QID_INNER) { 184214baf22SMaxim Mikityanskiy res = -EINVAL; 185214baf22SMaxim Mikityanskiy goto out; 186214baf22SMaxim Mikityanskiy } 187*28df4a01SMoshe Tal res = mlx5e_qid_from_qos(&htb->priv->channels, qid); 188214baf22SMaxim Mikityanskiy 189214baf22SMaxim Mikityanskiy out: 190214baf22SMaxim Mikityanskiy rcu_read_unlock(); 191214baf22SMaxim Mikityanskiy return res; 192214baf22SMaxim Mikityanskiy } 193214baf22SMaxim Mikityanskiy 19466d95936SMoshe Tal /* SQ lifecycle */ 19566d95936SMoshe Tal 196214baf22SMaxim Mikityanskiy static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) 197214baf22SMaxim Mikityanskiy { 198214baf22SMaxim Mikityanskiy struct mlx5e_params *params = &priv->channels.params; 199214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 200214baf22SMaxim Mikityanskiy struct mlx5e_channel *c; 201214baf22SMaxim Mikityanskiy int ix; 202214baf22SMaxim Mikityanskiy 203214baf22SMaxim Mikityanskiy ix = qid % params->num_channels; 204214baf22SMaxim Mikityanskiy qid /= params->num_channels; 205214baf22SMaxim Mikityanskiy c = priv->channels.c[ix]; 206214baf22SMaxim Mikityanskiy 207214baf22SMaxim Mikityanskiy qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 208214baf22SMaxim Mikityanskiy return mlx5e_state_dereference(priv, qos_sqs[qid]); 209214baf22SMaxim Mikityanskiy } 210214baf22SMaxim Mikityanskiy 211214baf22SMaxim Mikityanskiy static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, 212214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node) 213214baf22SMaxim Mikityanskiy { 214214baf22SMaxim Mikityanskiy struct mlx5e_create_cq_param ccp = {}; 215214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 216214baf22SMaxim Mikityanskiy struct mlx5e_sq_param param_sq; 217214baf22SMaxim Mikityanskiy struct mlx5e_cq_param param_cq; 218214baf22SMaxim Mikityanskiy int txq_ix, ix, qid, err = 0; 219214baf22SMaxim Mikityanskiy struct mlx5e_params *params; 220214baf22SMaxim Mikityanskiy struct mlx5e_channel *c; 221214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 222214baf22SMaxim Mikityanskiy 223214baf22SMaxim Mikityanskiy params = &chs->params; 224214baf22SMaxim Mikityanskiy 225214baf22SMaxim Mikityanskiy txq_ix = mlx5e_qid_from_qos(chs, node->qid); 226214baf22SMaxim Mikityanskiy 227db83f24dSMoshe Tal WARN_ON(node->qid > priv->htb_max_qos_sqs); 228db83f24dSMoshe Tal if (node->qid == priv->htb_max_qos_sqs) { 229214baf22SMaxim Mikityanskiy struct mlx5e_sq_stats *stats, **stats_list = NULL; 230214baf22SMaxim Mikityanskiy 231db83f24dSMoshe Tal if (priv->htb_max_qos_sqs == 0) { 232214baf22SMaxim Mikityanskiy stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev), 233214baf22SMaxim Mikityanskiy sizeof(*stats_list), 234214baf22SMaxim Mikityanskiy GFP_KERNEL); 235214baf22SMaxim Mikityanskiy if (!stats_list) 236214baf22SMaxim Mikityanskiy return -ENOMEM; 237214baf22SMaxim Mikityanskiy } 238214baf22SMaxim Mikityanskiy stats = kzalloc(sizeof(*stats), GFP_KERNEL); 239214baf22SMaxim Mikityanskiy if (!stats) { 240214baf22SMaxim Mikityanskiy kvfree(stats_list); 241214baf22SMaxim Mikityanskiy return -ENOMEM; 242214baf22SMaxim Mikityanskiy } 243214baf22SMaxim Mikityanskiy if (stats_list) 244db83f24dSMoshe Tal WRITE_ONCE(priv->htb_qos_sq_stats, stats_list); 245db83f24dSMoshe Tal WRITE_ONCE(priv->htb_qos_sq_stats[node->qid], stats); 246db83f24dSMoshe Tal /* Order htb_max_qos_sqs increment after writing the array pointer. 247214baf22SMaxim Mikityanskiy * Pairs with smp_load_acquire in en_stats.c. 248214baf22SMaxim Mikityanskiy */ 249db83f24dSMoshe Tal smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1); 250214baf22SMaxim Mikityanskiy } 251214baf22SMaxim Mikityanskiy 252214baf22SMaxim Mikityanskiy ix = node->qid % params->num_channels; 253214baf22SMaxim Mikityanskiy qid = node->qid / params->num_channels; 254214baf22SMaxim Mikityanskiy c = chs->c[ix]; 255214baf22SMaxim Mikityanskiy 256214baf22SMaxim Mikityanskiy qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 257214baf22SMaxim Mikityanskiy sq = kzalloc(sizeof(*sq), GFP_KERNEL); 258214baf22SMaxim Mikityanskiy 259214baf22SMaxim Mikityanskiy if (!sq) 260214baf22SMaxim Mikityanskiy return -ENOMEM; 261214baf22SMaxim Mikityanskiy 262214baf22SMaxim Mikityanskiy mlx5e_build_create_cq_param(&ccp, c); 263214baf22SMaxim Mikityanskiy 264214baf22SMaxim Mikityanskiy memset(¶m_sq, 0, sizeof(param_sq)); 265214baf22SMaxim Mikityanskiy memset(¶m_cq, 0, sizeof(param_cq)); 26689564920STariq Toukan mlx5e_build_sq_param(priv->mdev, params, ¶m_sq); 26789564920STariq Toukan mlx5e_build_tx_cq_param(priv->mdev, params, ¶m_cq); 268214baf22SMaxim Mikityanskiy err = mlx5e_open_cq(priv, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); 269214baf22SMaxim Mikityanskiy if (err) 270214baf22SMaxim Mikityanskiy goto err_free_sq; 271214baf22SMaxim Mikityanskiy err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, 272e0ee6891STariq Toukan ¶m_sq, sq, 0, node->hw_id, 273db83f24dSMoshe Tal priv->htb_qos_sq_stats[node->qid]); 274214baf22SMaxim Mikityanskiy if (err) 275214baf22SMaxim Mikityanskiy goto err_close_cq; 276214baf22SMaxim Mikityanskiy 277214baf22SMaxim Mikityanskiy rcu_assign_pointer(qos_sqs[qid], sq); 278214baf22SMaxim Mikityanskiy 279214baf22SMaxim Mikityanskiy return 0; 280214baf22SMaxim Mikityanskiy 281214baf22SMaxim Mikityanskiy err_close_cq: 282214baf22SMaxim Mikityanskiy mlx5e_close_cq(&sq->cq); 283214baf22SMaxim Mikityanskiy err_free_sq: 284214baf22SMaxim Mikityanskiy kfree(sq); 285214baf22SMaxim Mikityanskiy return err; 286214baf22SMaxim Mikityanskiy } 287214baf22SMaxim Mikityanskiy 288214baf22SMaxim Mikityanskiy static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) 289214baf22SMaxim Mikityanskiy { 290214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 29117c84cb4SMaxim Mikityanskiy u16 qid; 292214baf22SMaxim Mikityanskiy 293214baf22SMaxim Mikityanskiy sq = mlx5e_get_qos_sq(priv, node->qid); 294214baf22SMaxim Mikityanskiy 29517c84cb4SMaxim Mikityanskiy qid = mlx5e_qid_from_qos(&priv->channels, node->qid); 29617c84cb4SMaxim Mikityanskiy 29717c84cb4SMaxim Mikityanskiy /* If it's a new queue, it will be marked as started at this point. 29817c84cb4SMaxim Mikityanskiy * Stop it before updating txq2sq. 29917c84cb4SMaxim Mikityanskiy */ 30017c84cb4SMaxim Mikityanskiy mlx5e_tx_disable_queue(netdev_get_tx_queue(priv->netdev, qid)); 30117c84cb4SMaxim Mikityanskiy 30217c84cb4SMaxim Mikityanskiy priv->txq2sq[qid] = sq; 303214baf22SMaxim Mikityanskiy 304214baf22SMaxim Mikityanskiy /* Make the change to txq2sq visible before the queue is started. 305214baf22SMaxim Mikityanskiy * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 306214baf22SMaxim Mikityanskiy * which pairs with this barrier. 307214baf22SMaxim Mikityanskiy */ 308214baf22SMaxim Mikityanskiy smp_wmb(); 309214baf22SMaxim Mikityanskiy 310214baf22SMaxim Mikityanskiy qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node->qid); 311214baf22SMaxim Mikityanskiy mlx5e_activate_txqsq(sq); 312214baf22SMaxim Mikityanskiy } 313214baf22SMaxim Mikityanskiy 314214baf22SMaxim Mikityanskiy static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) 315214baf22SMaxim Mikityanskiy { 316214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 317214baf22SMaxim Mikityanskiy 318214baf22SMaxim Mikityanskiy sq = mlx5e_get_qos_sq(priv, qid); 319214baf22SMaxim Mikityanskiy if (!sq) /* Handle the case when the SQ failed to open. */ 320214baf22SMaxim Mikityanskiy return; 321214baf22SMaxim Mikityanskiy 322214baf22SMaxim Mikityanskiy qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid); 323214baf22SMaxim Mikityanskiy mlx5e_deactivate_txqsq(sq); 324214baf22SMaxim Mikityanskiy 325214baf22SMaxim Mikityanskiy priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL; 32617c84cb4SMaxim Mikityanskiy 32717c84cb4SMaxim Mikityanskiy /* Make the change to txq2sq visible before the queue is started again. 32817c84cb4SMaxim Mikityanskiy * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 32917c84cb4SMaxim Mikityanskiy * which pairs with this barrier. 33017c84cb4SMaxim Mikityanskiy */ 33117c84cb4SMaxim Mikityanskiy smp_wmb(); 332214baf22SMaxim Mikityanskiy } 333214baf22SMaxim Mikityanskiy 334214baf22SMaxim Mikityanskiy static void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) 335214baf22SMaxim Mikityanskiy { 336214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 337214baf22SMaxim Mikityanskiy struct mlx5e_params *params; 338214baf22SMaxim Mikityanskiy struct mlx5e_channel *c; 339214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 340214baf22SMaxim Mikityanskiy int ix; 341214baf22SMaxim Mikityanskiy 342214baf22SMaxim Mikityanskiy params = &priv->channels.params; 343214baf22SMaxim Mikityanskiy 344214baf22SMaxim Mikityanskiy ix = qid % params->num_channels; 345214baf22SMaxim Mikityanskiy qid /= params->num_channels; 346214baf22SMaxim Mikityanskiy c = priv->channels.c[ix]; 347214baf22SMaxim Mikityanskiy qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); 348214baf22SMaxim Mikityanskiy sq = rcu_replace_pointer(qos_sqs[qid], NULL, lockdep_is_held(&priv->state_lock)); 349214baf22SMaxim Mikityanskiy if (!sq) /* Handle the case when the SQ failed to open. */ 350214baf22SMaxim Mikityanskiy return; 351214baf22SMaxim Mikityanskiy 352214baf22SMaxim Mikityanskiy synchronize_rcu(); /* Sync with NAPI. */ 353214baf22SMaxim Mikityanskiy 354214baf22SMaxim Mikityanskiy mlx5e_close_txqsq(sq); 355214baf22SMaxim Mikityanskiy mlx5e_close_cq(&sq->cq); 356214baf22SMaxim Mikityanskiy kfree(sq); 357214baf22SMaxim Mikityanskiy } 358214baf22SMaxim Mikityanskiy 359214baf22SMaxim Mikityanskiy void mlx5e_qos_close_queues(struct mlx5e_channel *c) 360214baf22SMaxim Mikityanskiy { 361214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 362214baf22SMaxim Mikityanskiy int i; 363214baf22SMaxim Mikityanskiy 364214baf22SMaxim Mikityanskiy qos_sqs = rcu_replace_pointer(c->qos_sqs, NULL, lockdep_is_held(&c->priv->state_lock)); 365214baf22SMaxim Mikityanskiy if (!qos_sqs) 366214baf22SMaxim Mikityanskiy return; 367214baf22SMaxim Mikityanskiy synchronize_rcu(); /* Sync with NAPI. */ 368214baf22SMaxim Mikityanskiy 369214baf22SMaxim Mikityanskiy for (i = 0; i < c->qos_sqs_size; i++) { 370214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 371214baf22SMaxim Mikityanskiy 372214baf22SMaxim Mikityanskiy sq = mlx5e_state_dereference(c->priv, qos_sqs[i]); 373214baf22SMaxim Mikityanskiy if (!sq) /* Handle the case when the SQ failed to open. */ 374214baf22SMaxim Mikityanskiy continue; 375214baf22SMaxim Mikityanskiy 376214baf22SMaxim Mikityanskiy mlx5e_close_txqsq(sq); 377214baf22SMaxim Mikityanskiy mlx5e_close_cq(&sq->cq); 378214baf22SMaxim Mikityanskiy kfree(sq); 379214baf22SMaxim Mikityanskiy } 380214baf22SMaxim Mikityanskiy 381214baf22SMaxim Mikityanskiy kvfree(qos_sqs); 382214baf22SMaxim Mikityanskiy } 383214baf22SMaxim Mikityanskiy 384214baf22SMaxim Mikityanskiy static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) 385214baf22SMaxim Mikityanskiy { 386214baf22SMaxim Mikityanskiy int i; 387214baf22SMaxim Mikityanskiy 388214baf22SMaxim Mikityanskiy for (i = 0; i < chs->num; i++) 389214baf22SMaxim Mikityanskiy mlx5e_qos_close_queues(chs->c[i]); 390214baf22SMaxim Mikityanskiy } 391214baf22SMaxim Mikityanskiy 392214baf22SMaxim Mikityanskiy static int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 393214baf22SMaxim Mikityanskiy { 394214baf22SMaxim Mikityanskiy u16 qos_sqs_size; 395214baf22SMaxim Mikityanskiy int i; 396214baf22SMaxim Mikityanskiy 397214baf22SMaxim Mikityanskiy qos_sqs_size = DIV_ROUND_UP(mlx5e_qos_max_leaf_nodes(priv->mdev), chs->num); 398214baf22SMaxim Mikityanskiy 399214baf22SMaxim Mikityanskiy for (i = 0; i < chs->num; i++) { 400214baf22SMaxim Mikityanskiy struct mlx5e_txqsq **sqs; 401214baf22SMaxim Mikityanskiy 402214baf22SMaxim Mikityanskiy sqs = kvcalloc(qos_sqs_size, sizeof(struct mlx5e_txqsq *), GFP_KERNEL); 403214baf22SMaxim Mikityanskiy if (!sqs) 404214baf22SMaxim Mikityanskiy goto err_free; 405214baf22SMaxim Mikityanskiy 406214baf22SMaxim Mikityanskiy WRITE_ONCE(chs->c[i]->qos_sqs_size, qos_sqs_size); 407214baf22SMaxim Mikityanskiy smp_wmb(); /* Pairs with mlx5e_napi_poll. */ 408214baf22SMaxim Mikityanskiy rcu_assign_pointer(chs->c[i]->qos_sqs, sqs); 409214baf22SMaxim Mikityanskiy } 410214baf22SMaxim Mikityanskiy 411214baf22SMaxim Mikityanskiy return 0; 412214baf22SMaxim Mikityanskiy 413214baf22SMaxim Mikityanskiy err_free: 414214baf22SMaxim Mikityanskiy while (--i >= 0) { 415214baf22SMaxim Mikityanskiy struct mlx5e_txqsq **sqs; 416214baf22SMaxim Mikityanskiy 417214baf22SMaxim Mikityanskiy sqs = rcu_replace_pointer(chs->c[i]->qos_sqs, NULL, 418214baf22SMaxim Mikityanskiy lockdep_is_held(&priv->state_lock)); 419214baf22SMaxim Mikityanskiy 420214baf22SMaxim Mikityanskiy synchronize_rcu(); /* Sync with NAPI. */ 421214baf22SMaxim Mikityanskiy kvfree(sqs); 422214baf22SMaxim Mikityanskiy } 423214baf22SMaxim Mikityanskiy return -ENOMEM; 424214baf22SMaxim Mikityanskiy } 425214baf22SMaxim Mikityanskiy 426214baf22SMaxim Mikityanskiy int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 427214baf22SMaxim Mikityanskiy { 428214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node = NULL; 429214baf22SMaxim Mikityanskiy int bkt, err; 430214baf22SMaxim Mikityanskiy 431214baf22SMaxim Mikityanskiy err = mlx5e_qos_alloc_queues(priv, chs); 432214baf22SMaxim Mikityanskiy if (err) 433214baf22SMaxim Mikityanskiy return err; 434214baf22SMaxim Mikityanskiy 435aaffda6bSSaeed Mahameed hash_for_each(priv->htb->qos_tc2node, bkt, node, hnode) { 436214baf22SMaxim Mikityanskiy if (node->qid == MLX5E_QOS_QID_INNER) 437214baf22SMaxim Mikityanskiy continue; 438214baf22SMaxim Mikityanskiy err = mlx5e_open_qos_sq(priv, chs, node); 439214baf22SMaxim Mikityanskiy if (err) { 440214baf22SMaxim Mikityanskiy mlx5e_qos_close_all_queues(chs); 441214baf22SMaxim Mikityanskiy return err; 442214baf22SMaxim Mikityanskiy } 443214baf22SMaxim Mikityanskiy } 444214baf22SMaxim Mikityanskiy 445214baf22SMaxim Mikityanskiy return 0; 446214baf22SMaxim Mikityanskiy } 447214baf22SMaxim Mikityanskiy 448214baf22SMaxim Mikityanskiy void mlx5e_qos_activate_queues(struct mlx5e_priv *priv) 449214baf22SMaxim Mikityanskiy { 450214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node = NULL; 451214baf22SMaxim Mikityanskiy int bkt; 452214baf22SMaxim Mikityanskiy 453aaffda6bSSaeed Mahameed hash_for_each(priv->htb->qos_tc2node, bkt, node, hnode) { 454214baf22SMaxim Mikityanskiy if (node->qid == MLX5E_QOS_QID_INNER) 455214baf22SMaxim Mikityanskiy continue; 456214baf22SMaxim Mikityanskiy mlx5e_activate_qos_sq(priv, node); 457214baf22SMaxim Mikityanskiy } 458214baf22SMaxim Mikityanskiy } 459214baf22SMaxim Mikityanskiy 460214baf22SMaxim Mikityanskiy void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) 461214baf22SMaxim Mikityanskiy { 462214baf22SMaxim Mikityanskiy struct mlx5e_params *params = &c->priv->channels.params; 463214baf22SMaxim Mikityanskiy struct mlx5e_txqsq __rcu **qos_sqs; 464214baf22SMaxim Mikityanskiy int i; 465214baf22SMaxim Mikityanskiy 466214baf22SMaxim Mikityanskiy qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs); 467214baf22SMaxim Mikityanskiy if (!qos_sqs) 468214baf22SMaxim Mikityanskiy return; 469214baf22SMaxim Mikityanskiy 470214baf22SMaxim Mikityanskiy for (i = 0; i < c->qos_sqs_size; i++) { 471214baf22SMaxim Mikityanskiy u16 qid = params->num_channels * i + c->ix; 472214baf22SMaxim Mikityanskiy struct mlx5e_txqsq *sq; 473214baf22SMaxim Mikityanskiy 474214baf22SMaxim Mikityanskiy sq = mlx5e_state_dereference(c->priv, qos_sqs[i]); 475214baf22SMaxim Mikityanskiy if (!sq) /* Handle the case when the SQ failed to open. */ 476214baf22SMaxim Mikityanskiy continue; 477214baf22SMaxim Mikityanskiy 478214baf22SMaxim Mikityanskiy qos_dbg(c->mdev, "Deactivate QoS SQ qid %u\n", qid); 479214baf22SMaxim Mikityanskiy mlx5e_deactivate_txqsq(sq); 480214baf22SMaxim Mikityanskiy 481214baf22SMaxim Mikityanskiy /* The queue is disabled, no synchronization with datapath is needed. */ 482214baf22SMaxim Mikityanskiy c->priv->txq2sq[mlx5e_qid_from_qos(&c->priv->channels, qid)] = NULL; 483214baf22SMaxim Mikityanskiy } 484214baf22SMaxim Mikityanskiy } 485214baf22SMaxim Mikityanskiy 486214baf22SMaxim Mikityanskiy static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) 487214baf22SMaxim Mikityanskiy { 488214baf22SMaxim Mikityanskiy int i; 489214baf22SMaxim Mikityanskiy 490214baf22SMaxim Mikityanskiy for (i = 0; i < chs->num; i++) 491214baf22SMaxim Mikityanskiy mlx5e_qos_deactivate_queues(chs->c[i]); 492214baf22SMaxim Mikityanskiy } 493214baf22SMaxim Mikityanskiy 494efe31799SSaeed Mahameed /* HTB TC handlers */ 495214baf22SMaxim Mikityanskiy 496efe31799SSaeed Mahameed static int 497*28df4a01SMoshe Tal mlx5e_htb_root_add(struct mlx5e_htb *htb, u16 htb_maj_id, u16 htb_defcls, 498214baf22SMaxim Mikityanskiy struct netlink_ext_ack *extack) 499214baf22SMaxim Mikityanskiy { 500*28df4a01SMoshe Tal struct mlx5e_priv *priv = htb->priv; 501214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *root; 502214baf22SMaxim Mikityanskiy bool opened; 503214baf22SMaxim Mikityanskiy int err; 504214baf22SMaxim Mikityanskiy 505*28df4a01SMoshe Tal qos_dbg(htb->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls); 506214baf22SMaxim Mikityanskiy 507*28df4a01SMoshe Tal mlx5e_selq_prepare_htb(htb->selq, htb_maj_id, htb_defcls); 5084f8d1d3aSMoshe Tal 509214baf22SMaxim Mikityanskiy opened = test_bit(MLX5E_STATE_OPENED, &priv->state); 510214baf22SMaxim Mikityanskiy if (opened) { 511214baf22SMaxim Mikityanskiy err = mlx5e_qos_alloc_queues(priv, &priv->channels); 512214baf22SMaxim Mikityanskiy if (err) 5138bf30be7SMaxim Mikityanskiy goto err_cancel_selq; 514214baf22SMaxim Mikityanskiy } 515214baf22SMaxim Mikityanskiy 516*28df4a01SMoshe Tal root = mlx5e_sw_node_create_root(htb); 517214baf22SMaxim Mikityanskiy if (IS_ERR(root)) { 518214baf22SMaxim Mikityanskiy err = PTR_ERR(root); 519214baf22SMaxim Mikityanskiy goto err_free_queues; 520214baf22SMaxim Mikityanskiy } 521214baf22SMaxim Mikityanskiy 522*28df4a01SMoshe Tal err = mlx5_qos_create_root_node(htb->mdev, &root->hw_id); 523214baf22SMaxim Mikityanskiy if (err) { 524214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware."); 525214baf22SMaxim Mikityanskiy goto err_sw_node_delete; 526214baf22SMaxim Mikityanskiy } 527214baf22SMaxim Mikityanskiy 528*28df4a01SMoshe Tal mlx5e_selq_apply(htb->selq); 5298bf30be7SMaxim Mikityanskiy 530214baf22SMaxim Mikityanskiy return 0; 531214baf22SMaxim Mikityanskiy 532214baf22SMaxim Mikityanskiy err_sw_node_delete: 533*28df4a01SMoshe Tal mlx5e_sw_node_delete(htb, root); 534214baf22SMaxim Mikityanskiy 535214baf22SMaxim Mikityanskiy err_free_queues: 536214baf22SMaxim Mikityanskiy if (opened) 537214baf22SMaxim Mikityanskiy mlx5e_qos_close_all_queues(&priv->channels); 5388bf30be7SMaxim Mikityanskiy err_cancel_selq: 539*28df4a01SMoshe Tal mlx5e_selq_cancel(htb->selq); 540214baf22SMaxim Mikityanskiy return err; 541214baf22SMaxim Mikityanskiy } 542214baf22SMaxim Mikityanskiy 543*28df4a01SMoshe Tal static int mlx5e_htb_root_del(struct mlx5e_htb *htb) 544214baf22SMaxim Mikityanskiy { 545*28df4a01SMoshe Tal struct mlx5e_priv *priv = htb->priv; 546214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *root; 547214baf22SMaxim Mikityanskiy int err; 548214baf22SMaxim Mikityanskiy 549*28df4a01SMoshe Tal qos_dbg(htb->mdev, "TC_HTB_DESTROY\n"); 550214baf22SMaxim Mikityanskiy 5513ab45777SMaxim Mikityanskiy /* Wait until real_num_tx_queues is updated for mlx5e_select_queue, 5523ab45777SMaxim Mikityanskiy * so that we can safely switch to its non-HTB non-PTP fastpath. 5533ab45777SMaxim Mikityanskiy */ 5543ab45777SMaxim Mikityanskiy synchronize_net(); 5553ab45777SMaxim Mikityanskiy 556*28df4a01SMoshe Tal mlx5e_selq_prepare_htb(htb->selq, 0, 0); 557*28df4a01SMoshe Tal mlx5e_selq_apply(htb->selq); 5588bf30be7SMaxim Mikityanskiy 559*28df4a01SMoshe Tal root = mlx5e_sw_node_find(htb, MLX5E_HTB_CLASSID_ROOT); 560214baf22SMaxim Mikityanskiy if (!root) { 561*28df4a01SMoshe Tal qos_err(htb->mdev, "Failed to find the root node in the QoS tree\n"); 562214baf22SMaxim Mikityanskiy return -ENOENT; 563214baf22SMaxim Mikityanskiy } 564*28df4a01SMoshe Tal err = mlx5_qos_destroy_node(htb->mdev, root->hw_id); 565214baf22SMaxim Mikityanskiy if (err) 566*28df4a01SMoshe Tal qos_err(htb->mdev, "Failed to destroy root node %u, err = %d\n", 567214baf22SMaxim Mikityanskiy root->hw_id, err); 568*28df4a01SMoshe Tal mlx5e_sw_node_delete(htb, root); 569214baf22SMaxim Mikityanskiy 570214baf22SMaxim Mikityanskiy mlx5e_qos_deactivate_all_queues(&priv->channels); 571214baf22SMaxim Mikityanskiy mlx5e_qos_close_all_queues(&priv->channels); 572214baf22SMaxim Mikityanskiy 573214baf22SMaxim Mikityanskiy return err; 574214baf22SMaxim Mikityanskiy } 575214baf22SMaxim Mikityanskiy 576*28df4a01SMoshe Tal static int mlx5e_htb_convert_rate(struct mlx5e_htb *htb, u64 rate, 577214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *parent, u32 *bw_share) 578214baf22SMaxim Mikityanskiy { 579214baf22SMaxim Mikityanskiy u64 share = 0; 580214baf22SMaxim Mikityanskiy 581214baf22SMaxim Mikityanskiy while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw) 582214baf22SMaxim Mikityanskiy parent = parent->parent; 583214baf22SMaxim Mikityanskiy 584214baf22SMaxim Mikityanskiy if (parent->max_average_bw) 585214baf22SMaxim Mikityanskiy share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT), 586214baf22SMaxim Mikityanskiy parent->max_average_bw); 587214baf22SMaxim Mikityanskiy else 588214baf22SMaxim Mikityanskiy share = 101; 589214baf22SMaxim Mikityanskiy 590214baf22SMaxim Mikityanskiy *bw_share = share == 0 ? 1 : share > 100 ? 0 : share; 591214baf22SMaxim Mikityanskiy 592*28df4a01SMoshe Tal qos_dbg(htb->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n", 593214baf22SMaxim Mikityanskiy rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share); 594214baf22SMaxim Mikityanskiy 595214baf22SMaxim Mikityanskiy return 0; 596214baf22SMaxim Mikityanskiy } 597214baf22SMaxim Mikityanskiy 598*28df4a01SMoshe Tal static void mlx5e_htb_convert_ceil(struct mlx5e_htb *htb, u64 ceil, u32 *max_average_bw) 599214baf22SMaxim Mikityanskiy { 600736dfe4eSMaxim Mikityanskiy /* Hardware treats 0 as "unlimited", set at least 1. */ 601736dfe4eSMaxim Mikityanskiy *max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1); 602214baf22SMaxim Mikityanskiy 603*28df4a01SMoshe Tal qos_dbg(htb->mdev, "Convert: ceil %llu -> max_average_bw %u\n", 604214baf22SMaxim Mikityanskiy ceil, *max_average_bw); 605214baf22SMaxim Mikityanskiy } 606214baf22SMaxim Mikityanskiy 607efe31799SSaeed Mahameed static int 608*28df4a01SMoshe Tal mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid, 609214baf22SMaxim Mikityanskiy u32 parent_classid, u64 rate, u64 ceil, 610214baf22SMaxim Mikityanskiy struct netlink_ext_ack *extack) 611214baf22SMaxim Mikityanskiy { 612214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node, *parent; 613*28df4a01SMoshe Tal struct mlx5e_priv *priv = htb->priv; 614214baf22SMaxim Mikityanskiy int qid; 615214baf22SMaxim Mikityanskiy int err; 616214baf22SMaxim Mikityanskiy 617*28df4a01SMoshe Tal qos_dbg(htb->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n", 618214baf22SMaxim Mikityanskiy classid, parent_classid, rate, ceil); 619214baf22SMaxim Mikityanskiy 620*28df4a01SMoshe Tal qid = mlx5e_find_unused_qos_qid(htb); 621214baf22SMaxim Mikityanskiy if (qid < 0) { 622214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached."); 623214baf22SMaxim Mikityanskiy return qid; 624214baf22SMaxim Mikityanskiy } 625214baf22SMaxim Mikityanskiy 626*28df4a01SMoshe Tal parent = mlx5e_sw_node_find(htb, parent_classid); 627214baf22SMaxim Mikityanskiy if (!parent) 628214baf22SMaxim Mikityanskiy return -EINVAL; 629214baf22SMaxim Mikityanskiy 630*28df4a01SMoshe Tal node = mlx5e_sw_node_create_leaf(htb, classid, qid, parent); 631214baf22SMaxim Mikityanskiy if (IS_ERR(node)) 632214baf22SMaxim Mikityanskiy return PTR_ERR(node); 633214baf22SMaxim Mikityanskiy 634214baf22SMaxim Mikityanskiy node->rate = rate; 635*28df4a01SMoshe Tal mlx5e_htb_convert_rate(htb, rate, node->parent, &node->bw_share); 636*28df4a01SMoshe Tal mlx5e_htb_convert_ceil(htb, ceil, &node->max_average_bw); 637214baf22SMaxim Mikityanskiy 638*28df4a01SMoshe Tal err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->hw_id, 639214baf22SMaxim Mikityanskiy node->bw_share, node->max_average_bw, 640214baf22SMaxim Mikityanskiy &node->hw_id); 641214baf22SMaxim Mikityanskiy if (err) { 642214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); 643*28df4a01SMoshe Tal qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", 644214baf22SMaxim Mikityanskiy classid, err); 645*28df4a01SMoshe Tal mlx5e_sw_node_delete(htb, node); 646214baf22SMaxim Mikityanskiy return err; 647214baf22SMaxim Mikityanskiy } 648214baf22SMaxim Mikityanskiy 649214baf22SMaxim Mikityanskiy if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { 650214baf22SMaxim Mikityanskiy err = mlx5e_open_qos_sq(priv, &priv->channels, node); 651214baf22SMaxim Mikityanskiy if (err) { 652214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); 653*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", 654214baf22SMaxim Mikityanskiy classid, err); 655214baf22SMaxim Mikityanskiy } else { 656214baf22SMaxim Mikityanskiy mlx5e_activate_qos_sq(priv, node); 657214baf22SMaxim Mikityanskiy } 658214baf22SMaxim Mikityanskiy } 659214baf22SMaxim Mikityanskiy 660214baf22SMaxim Mikityanskiy return mlx5e_qid_from_qos(&priv->channels, node->qid); 661214baf22SMaxim Mikityanskiy } 662214baf22SMaxim Mikityanskiy 663efe31799SSaeed Mahameed static int 664*28df4a01SMoshe Tal mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid, 665214baf22SMaxim Mikityanskiy u64 rate, u64 ceil, struct netlink_ext_ack *extack) 666214baf22SMaxim Mikityanskiy { 667214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node, *child; 668*28df4a01SMoshe Tal struct mlx5e_priv *priv = htb->priv; 669214baf22SMaxim Mikityanskiy int err, tmp_err; 670214baf22SMaxim Mikityanskiy u32 new_hw_id; 671214baf22SMaxim Mikityanskiy u16 qid; 672214baf22SMaxim Mikityanskiy 673*28df4a01SMoshe Tal qos_dbg(htb->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n", 674214baf22SMaxim Mikityanskiy classid, child_classid, rate, ceil); 675214baf22SMaxim Mikityanskiy 676*28df4a01SMoshe Tal node = mlx5e_sw_node_find(htb, classid); 677214baf22SMaxim Mikityanskiy if (!node) 678214baf22SMaxim Mikityanskiy return -ENOENT; 679214baf22SMaxim Mikityanskiy 680*28df4a01SMoshe Tal err = mlx5_qos_create_inner_node(htb->mdev, node->parent->hw_id, 681214baf22SMaxim Mikityanskiy node->bw_share, node->max_average_bw, 682214baf22SMaxim Mikityanskiy &new_hw_id); 683214baf22SMaxim Mikityanskiy if (err) { 684214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node."); 685*28df4a01SMoshe Tal qos_err(htb->mdev, "Failed to create an inner node (class %04x), err = %d\n", 686214baf22SMaxim Mikityanskiy classid, err); 687214baf22SMaxim Mikityanskiy return err; 688214baf22SMaxim Mikityanskiy } 689214baf22SMaxim Mikityanskiy 690214baf22SMaxim Mikityanskiy /* Intentionally reuse the qid for the upcoming first child. */ 691*28df4a01SMoshe Tal child = mlx5e_sw_node_create_leaf(htb, child_classid, node->qid, node); 692214baf22SMaxim Mikityanskiy if (IS_ERR(child)) { 693214baf22SMaxim Mikityanskiy err = PTR_ERR(child); 694214baf22SMaxim Mikityanskiy goto err_destroy_hw_node; 695214baf22SMaxim Mikityanskiy } 696214baf22SMaxim Mikityanskiy 697214baf22SMaxim Mikityanskiy child->rate = rate; 698*28df4a01SMoshe Tal mlx5e_htb_convert_rate(htb, rate, node, &child->bw_share); 699*28df4a01SMoshe Tal mlx5e_htb_convert_ceil(htb, ceil, &child->max_average_bw); 700214baf22SMaxim Mikityanskiy 701*28df4a01SMoshe Tal err = mlx5_qos_create_leaf_node(htb->mdev, new_hw_id, child->bw_share, 702214baf22SMaxim Mikityanskiy child->max_average_bw, &child->hw_id); 703214baf22SMaxim Mikityanskiy if (err) { 704214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); 705*28df4a01SMoshe Tal qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", 706214baf22SMaxim Mikityanskiy classid, err); 707214baf22SMaxim Mikityanskiy goto err_delete_sw_node; 708214baf22SMaxim Mikityanskiy } 709214baf22SMaxim Mikityanskiy 710214baf22SMaxim Mikityanskiy /* No fail point. */ 711214baf22SMaxim Mikityanskiy 712214baf22SMaxim Mikityanskiy qid = node->qid; 713214baf22SMaxim Mikityanskiy /* Pairs with mlx5e_get_txq_by_classid. */ 714214baf22SMaxim Mikityanskiy WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); 715214baf22SMaxim Mikityanskiy 716214baf22SMaxim Mikityanskiy if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { 717214baf22SMaxim Mikityanskiy mlx5e_deactivate_qos_sq(priv, qid); 718214baf22SMaxim Mikityanskiy mlx5e_close_qos_sq(priv, qid); 719214baf22SMaxim Mikityanskiy } 720214baf22SMaxim Mikityanskiy 721*28df4a01SMoshe Tal err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); 722214baf22SMaxim Mikityanskiy if (err) /* Not fatal. */ 723*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", 724214baf22SMaxim Mikityanskiy node->hw_id, classid, err); 725214baf22SMaxim Mikityanskiy 726214baf22SMaxim Mikityanskiy node->hw_id = new_hw_id; 727214baf22SMaxim Mikityanskiy 728214baf22SMaxim Mikityanskiy if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { 729214baf22SMaxim Mikityanskiy err = mlx5e_open_qos_sq(priv, &priv->channels, child); 730214baf22SMaxim Mikityanskiy if (err) { 731214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); 732*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", 733214baf22SMaxim Mikityanskiy classid, err); 734214baf22SMaxim Mikityanskiy } else { 735214baf22SMaxim Mikityanskiy mlx5e_activate_qos_sq(priv, child); 736214baf22SMaxim Mikityanskiy } 737214baf22SMaxim Mikityanskiy } 738214baf22SMaxim Mikityanskiy 739214baf22SMaxim Mikityanskiy return 0; 740214baf22SMaxim Mikityanskiy 741214baf22SMaxim Mikityanskiy err_delete_sw_node: 742214baf22SMaxim Mikityanskiy child->qid = MLX5E_QOS_QID_INNER; 743*28df4a01SMoshe Tal mlx5e_sw_node_delete(htb, child); 744214baf22SMaxim Mikityanskiy 745214baf22SMaxim Mikityanskiy err_destroy_hw_node: 746*28df4a01SMoshe Tal tmp_err = mlx5_qos_destroy_node(htb->mdev, new_hw_id); 747214baf22SMaxim Mikityanskiy if (tmp_err) /* Not fatal. */ 748*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n", 749214baf22SMaxim Mikityanskiy new_hw_id, classid, tmp_err); 750214baf22SMaxim Mikityanskiy return err; 751214baf22SMaxim Mikityanskiy } 752214baf22SMaxim Mikityanskiy 753*28df4a01SMoshe Tal static struct mlx5e_qos_node *mlx5e_sw_node_find_by_qid(struct mlx5e_htb *htb, u16 qid) 754214baf22SMaxim Mikityanskiy { 755214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node = NULL; 756214baf22SMaxim Mikityanskiy int bkt; 757214baf22SMaxim Mikityanskiy 758*28df4a01SMoshe Tal hash_for_each(htb->qos_tc2node, bkt, node, hnode) 759214baf22SMaxim Mikityanskiy if (node->qid == qid) 760214baf22SMaxim Mikityanskiy break; 761214baf22SMaxim Mikityanskiy 762214baf22SMaxim Mikityanskiy return node; 763214baf22SMaxim Mikityanskiy } 764214baf22SMaxim Mikityanskiy 765214baf22SMaxim Mikityanskiy static void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) 766214baf22SMaxim Mikityanskiy { 767214baf22SMaxim Mikityanskiy qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid); 768214baf22SMaxim Mikityanskiy netdev_tx_reset_queue(txq); 769214baf22SMaxim Mikityanskiy netif_tx_start_queue(txq); 770214baf22SMaxim Mikityanskiy } 771214baf22SMaxim Mikityanskiy 772214baf22SMaxim Mikityanskiy static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) 773214baf22SMaxim Mikityanskiy { 774214baf22SMaxim Mikityanskiy struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid); 775214baf22SMaxim Mikityanskiy struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 776214baf22SMaxim Mikityanskiy 777214baf22SMaxim Mikityanskiy if (!qdisc) 778214baf22SMaxim Mikityanskiy return; 779214baf22SMaxim Mikityanskiy 780214baf22SMaxim Mikityanskiy spin_lock_bh(qdisc_lock(qdisc)); 781214baf22SMaxim Mikityanskiy qdisc_reset(qdisc); 782214baf22SMaxim Mikityanskiy spin_unlock_bh(qdisc_lock(qdisc)); 783214baf22SMaxim Mikityanskiy } 784214baf22SMaxim Mikityanskiy 785*28df4a01SMoshe Tal static int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid, 786ca49bfd9SMaxim Mikityanskiy struct netlink_ext_ack *extack) 787214baf22SMaxim Mikityanskiy { 788*28df4a01SMoshe Tal struct mlx5e_priv *priv = htb->priv; 789214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node; 790214baf22SMaxim Mikityanskiy struct netdev_queue *txq; 791214baf22SMaxim Mikityanskiy u16 qid, moved_qid; 792214baf22SMaxim Mikityanskiy bool opened; 793214baf22SMaxim Mikityanskiy int err; 794214baf22SMaxim Mikityanskiy 795*28df4a01SMoshe Tal qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid); 796214baf22SMaxim Mikityanskiy 797*28df4a01SMoshe Tal node = mlx5e_sw_node_find(htb, *classid); 798214baf22SMaxim Mikityanskiy if (!node) 799214baf22SMaxim Mikityanskiy return -ENOENT; 800214baf22SMaxim Mikityanskiy 801214baf22SMaxim Mikityanskiy /* Store qid for reuse. */ 802214baf22SMaxim Mikityanskiy qid = node->qid; 803214baf22SMaxim Mikityanskiy 804214baf22SMaxim Mikityanskiy opened = test_bit(MLX5E_STATE_OPENED, &priv->state); 805214baf22SMaxim Mikityanskiy if (opened) { 806*28df4a01SMoshe Tal txq = netdev_get_tx_queue(htb->netdev, 807214baf22SMaxim Mikityanskiy mlx5e_qid_from_qos(&priv->channels, qid)); 808214baf22SMaxim Mikityanskiy mlx5e_deactivate_qos_sq(priv, qid); 809214baf22SMaxim Mikityanskiy mlx5e_close_qos_sq(priv, qid); 810214baf22SMaxim Mikityanskiy } 811214baf22SMaxim Mikityanskiy 812*28df4a01SMoshe Tal err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); 813214baf22SMaxim Mikityanskiy if (err) /* Not fatal. */ 814*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", 815ca49bfd9SMaxim Mikityanskiy node->hw_id, *classid, err); 816214baf22SMaxim Mikityanskiy 817*28df4a01SMoshe Tal mlx5e_sw_node_delete(htb, node); 818214baf22SMaxim Mikityanskiy 819*28df4a01SMoshe Tal moved_qid = mlx5e_qos_cur_leaf_nodes(htb); 820214baf22SMaxim Mikityanskiy 821214baf22SMaxim Mikityanskiy if (moved_qid == 0) { 822214baf22SMaxim Mikityanskiy /* The last QoS SQ was just destroyed. */ 823214baf22SMaxim Mikityanskiy if (opened) 824214baf22SMaxim Mikityanskiy mlx5e_reactivate_qos_sq(priv, qid, txq); 825214baf22SMaxim Mikityanskiy return 0; 826214baf22SMaxim Mikityanskiy } 827214baf22SMaxim Mikityanskiy moved_qid--; 828214baf22SMaxim Mikityanskiy 829214baf22SMaxim Mikityanskiy if (moved_qid < qid) { 830214baf22SMaxim Mikityanskiy /* The highest QoS SQ was just destroyed. */ 831214baf22SMaxim Mikityanskiy WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u", 832214baf22SMaxim Mikityanskiy qid, moved_qid); 833214baf22SMaxim Mikityanskiy if (opened) 834214baf22SMaxim Mikityanskiy mlx5e_reactivate_qos_sq(priv, qid, txq); 835214baf22SMaxim Mikityanskiy return 0; 836214baf22SMaxim Mikityanskiy } 837214baf22SMaxim Mikityanskiy 838214baf22SMaxim Mikityanskiy WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid); 839*28df4a01SMoshe Tal qos_dbg(htb->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid); 840214baf22SMaxim Mikityanskiy 841*28df4a01SMoshe Tal node = mlx5e_sw_node_find_by_qid(htb, moved_qid); 842214baf22SMaxim Mikityanskiy WARN(!node, "Could not find a node with qid %u to move to queue %u", 843214baf22SMaxim Mikityanskiy moved_qid, qid); 844214baf22SMaxim Mikityanskiy 845214baf22SMaxim Mikityanskiy /* Stop traffic to the old queue. */ 846214baf22SMaxim Mikityanskiy WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); 847aaffda6bSSaeed Mahameed __clear_bit(moved_qid, priv->htb->qos_used_qids); 848214baf22SMaxim Mikityanskiy 849214baf22SMaxim Mikityanskiy if (opened) { 850*28df4a01SMoshe Tal txq = netdev_get_tx_queue(htb->netdev, 851214baf22SMaxim Mikityanskiy mlx5e_qid_from_qos(&priv->channels, moved_qid)); 852214baf22SMaxim Mikityanskiy mlx5e_deactivate_qos_sq(priv, moved_qid); 853214baf22SMaxim Mikityanskiy mlx5e_close_qos_sq(priv, moved_qid); 854214baf22SMaxim Mikityanskiy } 855214baf22SMaxim Mikityanskiy 856214baf22SMaxim Mikityanskiy /* Prevent packets from the old class from getting into the new one. */ 857*28df4a01SMoshe Tal mlx5e_reset_qdisc(htb->netdev, moved_qid); 858214baf22SMaxim Mikityanskiy 859*28df4a01SMoshe Tal __set_bit(qid, htb->qos_used_qids); 860214baf22SMaxim Mikityanskiy WRITE_ONCE(node->qid, qid); 861214baf22SMaxim Mikityanskiy 862214baf22SMaxim Mikityanskiy if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { 863214baf22SMaxim Mikityanskiy err = mlx5e_open_qos_sq(priv, &priv->channels, node); 864214baf22SMaxim Mikityanskiy if (err) { 865214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); 866*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n", 867214baf22SMaxim Mikityanskiy node->classid, moved_qid, qid, err); 868214baf22SMaxim Mikityanskiy } else { 869214baf22SMaxim Mikityanskiy mlx5e_activate_qos_sq(priv, node); 870214baf22SMaxim Mikityanskiy } 871214baf22SMaxim Mikityanskiy } 872214baf22SMaxim Mikityanskiy 873214baf22SMaxim Mikityanskiy mlx5e_update_tx_netdev_queues(priv); 874214baf22SMaxim Mikityanskiy if (opened) 875214baf22SMaxim Mikityanskiy mlx5e_reactivate_qos_sq(priv, moved_qid, txq); 876214baf22SMaxim Mikityanskiy 877ca49bfd9SMaxim Mikityanskiy *classid = node->classid; 878214baf22SMaxim Mikityanskiy return 0; 879214baf22SMaxim Mikityanskiy } 880214baf22SMaxim Mikityanskiy 881efe31799SSaeed Mahameed static int 882*28df4a01SMoshe Tal mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force, 883214baf22SMaxim Mikityanskiy struct netlink_ext_ack *extack) 884214baf22SMaxim Mikityanskiy { 885214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node, *parent; 886*28df4a01SMoshe Tal struct mlx5e_priv *priv = htb->priv; 887214baf22SMaxim Mikityanskiy u32 old_hw_id, new_hw_id; 888214baf22SMaxim Mikityanskiy int err, saved_err = 0; 889214baf22SMaxim Mikityanskiy u16 qid; 890214baf22SMaxim Mikityanskiy 891*28df4a01SMoshe Tal qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n", 892214baf22SMaxim Mikityanskiy force ? "_FORCE" : "", classid); 893214baf22SMaxim Mikityanskiy 894*28df4a01SMoshe Tal node = mlx5e_sw_node_find(htb, classid); 895214baf22SMaxim Mikityanskiy if (!node) 896214baf22SMaxim Mikityanskiy return -ENOENT; 897214baf22SMaxim Mikityanskiy 898*28df4a01SMoshe Tal err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->parent->hw_id, 899214baf22SMaxim Mikityanskiy node->parent->bw_share, 900214baf22SMaxim Mikityanskiy node->parent->max_average_bw, 901214baf22SMaxim Mikityanskiy &new_hw_id); 902214baf22SMaxim Mikityanskiy if (err) { 903214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); 904*28df4a01SMoshe Tal qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", 905214baf22SMaxim Mikityanskiy classid, err); 906214baf22SMaxim Mikityanskiy if (!force) 907214baf22SMaxim Mikityanskiy return err; 908214baf22SMaxim Mikityanskiy saved_err = err; 909214baf22SMaxim Mikityanskiy } 910214baf22SMaxim Mikityanskiy 911214baf22SMaxim Mikityanskiy /* Store qid for reuse and prevent clearing the bit. */ 912214baf22SMaxim Mikityanskiy qid = node->qid; 913214baf22SMaxim Mikityanskiy /* Pairs with mlx5e_get_txq_by_classid. */ 914214baf22SMaxim Mikityanskiy WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); 915214baf22SMaxim Mikityanskiy 916214baf22SMaxim Mikityanskiy if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { 917214baf22SMaxim Mikityanskiy mlx5e_deactivate_qos_sq(priv, qid); 918214baf22SMaxim Mikityanskiy mlx5e_close_qos_sq(priv, qid); 919214baf22SMaxim Mikityanskiy } 920214baf22SMaxim Mikityanskiy 921214baf22SMaxim Mikityanskiy /* Prevent packets from the old class from getting into the new one. */ 922*28df4a01SMoshe Tal mlx5e_reset_qdisc(htb->netdev, qid); 923214baf22SMaxim Mikityanskiy 924*28df4a01SMoshe Tal err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); 925214baf22SMaxim Mikityanskiy if (err) /* Not fatal. */ 926*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", 927214baf22SMaxim Mikityanskiy node->hw_id, classid, err); 928214baf22SMaxim Mikityanskiy 929214baf22SMaxim Mikityanskiy parent = node->parent; 930*28df4a01SMoshe Tal mlx5e_sw_node_delete(htb, node); 931214baf22SMaxim Mikityanskiy 932214baf22SMaxim Mikityanskiy node = parent; 933214baf22SMaxim Mikityanskiy WRITE_ONCE(node->qid, qid); 934214baf22SMaxim Mikityanskiy 935214baf22SMaxim Mikityanskiy /* Early return on error in force mode. Parent will still be an inner 936214baf22SMaxim Mikityanskiy * node to be deleted by a following delete operation. 937214baf22SMaxim Mikityanskiy */ 938214baf22SMaxim Mikityanskiy if (saved_err) 939214baf22SMaxim Mikityanskiy return saved_err; 940214baf22SMaxim Mikityanskiy 941214baf22SMaxim Mikityanskiy old_hw_id = node->hw_id; 942214baf22SMaxim Mikityanskiy node->hw_id = new_hw_id; 943214baf22SMaxim Mikityanskiy 944214baf22SMaxim Mikityanskiy if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { 945214baf22SMaxim Mikityanskiy err = mlx5e_open_qos_sq(priv, &priv->channels, node); 946214baf22SMaxim Mikityanskiy if (err) { 947214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); 948*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", 949214baf22SMaxim Mikityanskiy classid, err); 950214baf22SMaxim Mikityanskiy } else { 951214baf22SMaxim Mikityanskiy mlx5e_activate_qos_sq(priv, node); 952214baf22SMaxim Mikityanskiy } 953214baf22SMaxim Mikityanskiy } 954214baf22SMaxim Mikityanskiy 955*28df4a01SMoshe Tal err = mlx5_qos_destroy_node(htb->mdev, old_hw_id); 956214baf22SMaxim Mikityanskiy if (err) /* Not fatal. */ 957*28df4a01SMoshe Tal qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", 958214baf22SMaxim Mikityanskiy node->hw_id, classid, err); 959214baf22SMaxim Mikityanskiy 960214baf22SMaxim Mikityanskiy return 0; 961214baf22SMaxim Mikityanskiy } 962214baf22SMaxim Mikityanskiy 963efe31799SSaeed Mahameed static int 964*28df4a01SMoshe Tal mlx5e_qos_update_children(struct mlx5e_htb *htb, struct mlx5e_qos_node *node, 965214baf22SMaxim Mikityanskiy struct netlink_ext_ack *extack) 966214baf22SMaxim Mikityanskiy { 967214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *child; 968214baf22SMaxim Mikityanskiy int err = 0; 969214baf22SMaxim Mikityanskiy int bkt; 970214baf22SMaxim Mikityanskiy 971*28df4a01SMoshe Tal hash_for_each(htb->qos_tc2node, bkt, child, hnode) { 972214baf22SMaxim Mikityanskiy u32 old_bw_share = child->bw_share; 973214baf22SMaxim Mikityanskiy int err_one; 974214baf22SMaxim Mikityanskiy 975214baf22SMaxim Mikityanskiy if (child->parent != node) 976214baf22SMaxim Mikityanskiy continue; 977214baf22SMaxim Mikityanskiy 978*28df4a01SMoshe Tal mlx5e_htb_convert_rate(htb, child->rate, node, &child->bw_share); 979214baf22SMaxim Mikityanskiy if (child->bw_share == old_bw_share) 980214baf22SMaxim Mikityanskiy continue; 981214baf22SMaxim Mikityanskiy 982*28df4a01SMoshe Tal err_one = mlx5_qos_update_node(htb->mdev, child->hw_id, child->bw_share, 983214baf22SMaxim Mikityanskiy child->max_average_bw, child->hw_id); 984214baf22SMaxim Mikityanskiy if (!err && err_one) { 985214baf22SMaxim Mikityanskiy err = err_one; 986214baf22SMaxim Mikityanskiy 987214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node."); 988*28df4a01SMoshe Tal qos_err(htb->mdev, "Failed to modify a child node (class %04x), err = %d\n", 989214baf22SMaxim Mikityanskiy node->classid, err); 990214baf22SMaxim Mikityanskiy } 991214baf22SMaxim Mikityanskiy } 992214baf22SMaxim Mikityanskiy 993214baf22SMaxim Mikityanskiy return err; 994214baf22SMaxim Mikityanskiy } 995214baf22SMaxim Mikityanskiy 996efe31799SSaeed Mahameed static int 997*28df4a01SMoshe Tal mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil, 998214baf22SMaxim Mikityanskiy struct netlink_ext_ack *extack) 999214baf22SMaxim Mikityanskiy { 1000214baf22SMaxim Mikityanskiy u32 bw_share, max_average_bw; 1001214baf22SMaxim Mikityanskiy struct mlx5e_qos_node *node; 1002214baf22SMaxim Mikityanskiy bool ceil_changed = false; 1003214baf22SMaxim Mikityanskiy int err; 1004214baf22SMaxim Mikityanskiy 1005*28df4a01SMoshe Tal qos_dbg(htb->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n", 1006214baf22SMaxim Mikityanskiy classid, rate, ceil); 1007214baf22SMaxim Mikityanskiy 1008*28df4a01SMoshe Tal node = mlx5e_sw_node_find(htb, classid); 1009214baf22SMaxim Mikityanskiy if (!node) 1010214baf22SMaxim Mikityanskiy return -ENOENT; 1011214baf22SMaxim Mikityanskiy 1012214baf22SMaxim Mikityanskiy node->rate = rate; 1013*28df4a01SMoshe Tal mlx5e_htb_convert_rate(htb, rate, node->parent, &bw_share); 1014*28df4a01SMoshe Tal mlx5e_htb_convert_ceil(htb, ceil, &max_average_bw); 1015214baf22SMaxim Mikityanskiy 1016*28df4a01SMoshe Tal err = mlx5_qos_update_node(htb->mdev, node->parent->hw_id, bw_share, 1017214baf22SMaxim Mikityanskiy max_average_bw, node->hw_id); 1018214baf22SMaxim Mikityanskiy if (err) { 1019214baf22SMaxim Mikityanskiy NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node."); 1020*28df4a01SMoshe Tal qos_err(htb->mdev, "Failed to modify a node (class %04x), err = %d\n", 1021214baf22SMaxim Mikityanskiy classid, err); 1022214baf22SMaxim Mikityanskiy return err; 1023214baf22SMaxim Mikityanskiy } 1024214baf22SMaxim Mikityanskiy 1025214baf22SMaxim Mikityanskiy if (max_average_bw != node->max_average_bw) 1026214baf22SMaxim Mikityanskiy ceil_changed = true; 1027214baf22SMaxim Mikityanskiy 1028214baf22SMaxim Mikityanskiy node->bw_share = bw_share; 1029214baf22SMaxim Mikityanskiy node->max_average_bw = max_average_bw; 1030214baf22SMaxim Mikityanskiy 1031214baf22SMaxim Mikityanskiy if (ceil_changed) 1032*28df4a01SMoshe Tal err = mlx5e_qos_update_children(htb, node, extack); 1033214baf22SMaxim Mikityanskiy 1034214baf22SMaxim Mikityanskiy return err; 1035214baf22SMaxim Mikityanskiy } 103680743c4fSTariq Toukan 1037efe31799SSaeed Mahameed /* HTB API */ 1038aaffda6bSSaeed Mahameed 1039aaffda6bSSaeed Mahameed static struct mlx5e_htb *mlx5e_htb_alloc(void) 1040aaffda6bSSaeed Mahameed { 1041aaffda6bSSaeed Mahameed return kvzalloc(sizeof(struct mlx5e_htb), GFP_KERNEL); 1042aaffda6bSSaeed Mahameed } 1043aaffda6bSSaeed Mahameed 1044aaffda6bSSaeed Mahameed static void mlx5e_htb_free(struct mlx5e_htb *htb) 1045aaffda6bSSaeed Mahameed { 1046aaffda6bSSaeed Mahameed kvfree(htb); 1047aaffda6bSSaeed Mahameed } 1048aaffda6bSSaeed Mahameed 1049*28df4a01SMoshe Tal /* HTB API */ 1050aaffda6bSSaeed Mahameed 1051*28df4a01SMoshe Tal static int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt, 1052*28df4a01SMoshe Tal struct net_device *netdev, struct mlx5_core_dev *mdev, 1053*28df4a01SMoshe Tal struct mlx5e_selq *selq, struct mlx5e_priv *priv) 1054*28df4a01SMoshe Tal { 1055*28df4a01SMoshe Tal htb->mdev = mdev; 1056*28df4a01SMoshe Tal htb->netdev = netdev; 1057*28df4a01SMoshe Tal htb->selq = selq; 1058*28df4a01SMoshe Tal htb->priv = priv; 1059*28df4a01SMoshe Tal hash_init(htb->qos_tc2node); 1060*28df4a01SMoshe Tal return mlx5e_htb_root_add(htb, htb_qopt->parent_classid, htb_qopt->classid, 1061*28df4a01SMoshe Tal htb_qopt->extack); 1062aaffda6bSSaeed Mahameed } 1063aaffda6bSSaeed Mahameed 1064*28df4a01SMoshe Tal static void mlx5e_htb_cleanup(struct mlx5e_htb *htb) 1065aaffda6bSSaeed Mahameed { 1066*28df4a01SMoshe Tal mlx5e_htb_root_del(htb); 1067aaffda6bSSaeed Mahameed } 1068aaffda6bSSaeed Mahameed 1069*28df4a01SMoshe Tal int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt) 1070efe31799SSaeed Mahameed { 1071*28df4a01SMoshe Tal struct mlx5e_htb *htb = priv->htb; 1072efe31799SSaeed Mahameed int res; 1073efe31799SSaeed Mahameed 1074*28df4a01SMoshe Tal if (!htb && htb_qopt->command != TC_HTB_CREATE) 1075aaffda6bSSaeed Mahameed return -EINVAL; 1076aaffda6bSSaeed Mahameed 1077*28df4a01SMoshe Tal switch (htb_qopt->command) { 1078efe31799SSaeed Mahameed case TC_HTB_CREATE: 1079aaffda6bSSaeed Mahameed if (!mlx5_qos_is_supported(priv->mdev)) { 1080*28df4a01SMoshe Tal NL_SET_ERR_MSG_MOD(htb_qopt->extack, 1081aaffda6bSSaeed Mahameed "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); 1082aaffda6bSSaeed Mahameed return -EOPNOTSUPP; 1083aaffda6bSSaeed Mahameed } 1084aaffda6bSSaeed Mahameed priv->htb = mlx5e_htb_alloc(); 1085*28df4a01SMoshe Tal htb = priv->htb; 1086*28df4a01SMoshe Tal if (!htb) 1087aaffda6bSSaeed Mahameed return -ENOMEM; 1088*28df4a01SMoshe Tal res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv); 1089aaffda6bSSaeed Mahameed if (res) { 1090*28df4a01SMoshe Tal mlx5e_htb_free(htb); 1091aaffda6bSSaeed Mahameed priv->htb = NULL; 1092aaffda6bSSaeed Mahameed } 1093aaffda6bSSaeed Mahameed return res; 1094efe31799SSaeed Mahameed case TC_HTB_DESTROY: 1095*28df4a01SMoshe Tal mlx5e_htb_cleanup(htb); 1096*28df4a01SMoshe Tal mlx5e_htb_free(htb); 1097aaffda6bSSaeed Mahameed priv->htb = NULL; 1098aaffda6bSSaeed Mahameed return 0; 1099efe31799SSaeed Mahameed case TC_HTB_LEAF_ALLOC_QUEUE: 1100*28df4a01SMoshe Tal res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid, 1101*28df4a01SMoshe Tal htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); 1102efe31799SSaeed Mahameed if (res < 0) 1103efe31799SSaeed Mahameed return res; 1104*28df4a01SMoshe Tal htb_qopt->qid = res; 1105efe31799SSaeed Mahameed return 0; 1106efe31799SSaeed Mahameed case TC_HTB_LEAF_TO_INNER: 1107*28df4a01SMoshe Tal return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid, 1108*28df4a01SMoshe Tal htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); 1109efe31799SSaeed Mahameed case TC_HTB_LEAF_DEL: 1110*28df4a01SMoshe Tal return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack); 1111efe31799SSaeed Mahameed case TC_HTB_LEAF_DEL_LAST: 1112efe31799SSaeed Mahameed case TC_HTB_LEAF_DEL_LAST_FORCE: 1113*28df4a01SMoshe Tal return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid, 1114*28df4a01SMoshe Tal htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE, 1115*28df4a01SMoshe Tal htb_qopt->extack); 1116efe31799SSaeed Mahameed case TC_HTB_NODE_MODIFY: 1117*28df4a01SMoshe Tal return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil, 1118*28df4a01SMoshe Tal htb_qopt->extack); 1119efe31799SSaeed Mahameed case TC_HTB_LEAF_QUERY_QUEUE: 1120*28df4a01SMoshe Tal res = mlx5e_get_txq_by_classid(htb, htb_qopt->classid); 1121efe31799SSaeed Mahameed if (res < 0) 1122efe31799SSaeed Mahameed return res; 1123*28df4a01SMoshe Tal htb_qopt->qid = res; 1124efe31799SSaeed Mahameed return 0; 1125efe31799SSaeed Mahameed default: 1126efe31799SSaeed Mahameed return -EOPNOTSUPP; 1127efe31799SSaeed Mahameed } 1128efe31799SSaeed Mahameed } 1129efe31799SSaeed Mahameed 113080743c4fSTariq Toukan struct mlx5e_mqprio_rl { 113180743c4fSTariq Toukan struct mlx5_core_dev *mdev; 113280743c4fSTariq Toukan u32 root_id; 113380743c4fSTariq Toukan u32 *leaves_id; 113480743c4fSTariq Toukan u8 num_tc; 113580743c4fSTariq Toukan }; 113680743c4fSTariq Toukan 113780743c4fSTariq Toukan struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_alloc(void) 113880743c4fSTariq Toukan { 113980743c4fSTariq Toukan return kvzalloc(sizeof(struct mlx5e_mqprio_rl), GFP_KERNEL); 114080743c4fSTariq Toukan } 114180743c4fSTariq Toukan 114280743c4fSTariq Toukan void mlx5e_mqprio_rl_free(struct mlx5e_mqprio_rl *rl) 114380743c4fSTariq Toukan { 114480743c4fSTariq Toukan kvfree(rl); 114580743c4fSTariq Toukan } 114680743c4fSTariq Toukan 114780743c4fSTariq Toukan int mlx5e_mqprio_rl_init(struct mlx5e_mqprio_rl *rl, struct mlx5_core_dev *mdev, u8 num_tc, 114880743c4fSTariq Toukan u64 max_rate[]) 114980743c4fSTariq Toukan { 115080743c4fSTariq Toukan int err; 115180743c4fSTariq Toukan int tc; 115280743c4fSTariq Toukan 115380743c4fSTariq Toukan if (!mlx5_qos_is_supported(mdev)) { 115480743c4fSTariq Toukan qos_warn(mdev, "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); 115580743c4fSTariq Toukan return -EOPNOTSUPP; 115680743c4fSTariq Toukan } 115780743c4fSTariq Toukan if (num_tc > mlx5e_qos_max_leaf_nodes(mdev)) 115880743c4fSTariq Toukan return -EINVAL; 115980743c4fSTariq Toukan 116080743c4fSTariq Toukan rl->mdev = mdev; 116180743c4fSTariq Toukan rl->num_tc = num_tc; 116280743c4fSTariq Toukan rl->leaves_id = kvcalloc(num_tc, sizeof(*rl->leaves_id), GFP_KERNEL); 116380743c4fSTariq Toukan if (!rl->leaves_id) 116480743c4fSTariq Toukan return -ENOMEM; 116580743c4fSTariq Toukan 116680743c4fSTariq Toukan err = mlx5_qos_create_root_node(mdev, &rl->root_id); 116780743c4fSTariq Toukan if (err) 116880743c4fSTariq Toukan goto err_free_leaves; 116980743c4fSTariq Toukan 117080743c4fSTariq Toukan qos_dbg(mdev, "Root created, id %#x\n", rl->root_id); 117180743c4fSTariq Toukan 117280743c4fSTariq Toukan for (tc = 0; tc < num_tc; tc++) { 117380743c4fSTariq Toukan u32 max_average_bw; 117480743c4fSTariq Toukan 117580743c4fSTariq Toukan max_average_bw = mlx5e_qos_bytes2mbits(mdev, max_rate[tc]); 117680743c4fSTariq Toukan err = mlx5_qos_create_leaf_node(mdev, rl->root_id, 0, max_average_bw, 117780743c4fSTariq Toukan &rl->leaves_id[tc]); 117880743c4fSTariq Toukan if (err) 117980743c4fSTariq Toukan goto err_destroy_leaves; 118080743c4fSTariq Toukan 118180743c4fSTariq Toukan qos_dbg(mdev, "Leaf[%d] created, id %#x, max average bw %u Mbits/sec\n", 118280743c4fSTariq Toukan tc, rl->leaves_id[tc], max_average_bw); 118380743c4fSTariq Toukan } 118480743c4fSTariq Toukan return 0; 118580743c4fSTariq Toukan 118680743c4fSTariq Toukan err_destroy_leaves: 118780743c4fSTariq Toukan while (--tc >= 0) 118880743c4fSTariq Toukan mlx5_qos_destroy_node(mdev, rl->leaves_id[tc]); 118980743c4fSTariq Toukan mlx5_qos_destroy_node(mdev, rl->root_id); 119080743c4fSTariq Toukan err_free_leaves: 119180743c4fSTariq Toukan kvfree(rl->leaves_id); 119280743c4fSTariq Toukan return err; 119380743c4fSTariq Toukan } 119480743c4fSTariq Toukan 119580743c4fSTariq Toukan void mlx5e_mqprio_rl_cleanup(struct mlx5e_mqprio_rl *rl) 119680743c4fSTariq Toukan { 119780743c4fSTariq Toukan int tc; 119880743c4fSTariq Toukan 119980743c4fSTariq Toukan for (tc = 0; tc < rl->num_tc; tc++) 120080743c4fSTariq Toukan mlx5_qos_destroy_node(rl->mdev, rl->leaves_id[tc]); 120180743c4fSTariq Toukan mlx5_qos_destroy_node(rl->mdev, rl->root_id); 120280743c4fSTariq Toukan kvfree(rl->leaves_id); 120380743c4fSTariq Toukan } 120480743c4fSTariq Toukan 120580743c4fSTariq Toukan int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_id) 120680743c4fSTariq Toukan { 120780743c4fSTariq Toukan if (tc >= rl->num_tc) 120880743c4fSTariq Toukan return -EINVAL; 120980743c4fSTariq Toukan 121080743c4fSTariq Toukan *hw_id = rl->leaves_id[tc]; 121180743c4fSTariq Toukan return 0; 121280743c4fSTariq Toukan } 1213efe31799SSaeed Mahameed 1214