1e8f887acSAmir Vadai /* 2e8f887acSAmir Vadai * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3e8f887acSAmir Vadai * 4e8f887acSAmir Vadai * This software is available to you under a choice of one of two 5e8f887acSAmir Vadai * licenses. You may choose to be licensed under the terms of the GNU 6e8f887acSAmir Vadai * General Public License (GPL) Version 2, available from the file 7e8f887acSAmir Vadai * COPYING in the main directory of this source tree, or the 8e8f887acSAmir Vadai * OpenIB.org BSD license below: 9e8f887acSAmir Vadai * 10e8f887acSAmir Vadai * Redistribution and use in source and binary forms, with or 11e8f887acSAmir Vadai * without modification, are permitted provided that the following 12e8f887acSAmir Vadai * conditions are met: 13e8f887acSAmir Vadai * 14e8f887acSAmir Vadai * - Redistributions of source code must retain the above 15e8f887acSAmir Vadai * copyright notice, this list of conditions and the following 16e8f887acSAmir Vadai * disclaimer. 17e8f887acSAmir Vadai * 18e8f887acSAmir Vadai * - Redistributions in binary form must reproduce the above 19e8f887acSAmir Vadai * copyright notice, this list of conditions and the following 20e8f887acSAmir Vadai * disclaimer in the documentation and/or other materials 21e8f887acSAmir Vadai * provided with the distribution. 22e8f887acSAmir Vadai * 23e8f887acSAmir Vadai * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e8f887acSAmir Vadai * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e8f887acSAmir Vadai * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e8f887acSAmir Vadai * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e8f887acSAmir Vadai * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e8f887acSAmir Vadai * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e8f887acSAmir Vadai * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e8f887acSAmir Vadai * SOFTWARE. 31e8f887acSAmir Vadai */ 32e8f887acSAmir Vadai 33e3a2b7edSAmir Vadai #include <net/flow_dissector.h> 343f7d0eb4SOr Gerlitz #include <net/sch_generic.h> 35e3a2b7edSAmir Vadai #include <net/pkt_cls.h> 36e3a2b7edSAmir Vadai #include <net/tc_act/tc_gact.h> 3712185a9fSAmir Vadai #include <net/tc_act/tc_skbedit.h> 38e8f887acSAmir Vadai #include <linux/mlx5/fs.h> 39e8f887acSAmir Vadai #include <linux/mlx5/device.h> 40e8f887acSAmir Vadai #include <linux/rhashtable.h> 4103a9d11eSOr Gerlitz #include <net/switchdev.h> 4203a9d11eSOr Gerlitz #include <net/tc_act/tc_mirred.h> 43776b12b6SOr Gerlitz #include <net/tc_act/tc_vlan.h> 44bbd00f7eSHadar Hen Zion #include <net/tc_act/tc_tunnel_key.h> 45d79b6df6SOr Gerlitz #include <net/tc_act/tc_pedit.h> 4626c02749SOr Gerlitz #include <net/tc_act/tc_csum.h> 47f6dfb4c3SHadar Hen Zion #include <net/arp.h> 48e8f887acSAmir Vadai #include "en.h" 491d447a39SSaeed Mahameed #include "en_rep.h" 50232c0013SHadar Hen Zion #include "en_tc.h" 5103a9d11eSOr Gerlitz #include "eswitch.h" 523f6d08d1SOr Gerlitz #include "fs_core.h" 532c81bfd5SHuy Nguyen #include "en/port.h" 54101f4de9SOz Shlomo #include "en/tc_tun.h" 55e8f887acSAmir Vadai 563bc4b7bfSOr Gerlitz struct mlx5_nic_flow_attr { 573bc4b7bfSOr Gerlitz u32 action; 583bc4b7bfSOr Gerlitz u32 flow_tag; 592f4fe4caSOr Gerlitz u32 mod_hdr_id; 605c65c564SOr Gerlitz u32 hairpin_tirn; 6138aa51c1SOr Gerlitz u8 match_level; 623f6d08d1SOr Gerlitz struct mlx5_flow_table *hairpin_ft; 63b8aee822SMark Bloch struct mlx5_fc *counter; 643bc4b7bfSOr Gerlitz }; 653bc4b7bfSOr Gerlitz 6660bd4af8SOr Gerlitz #define MLX5E_TC_FLOW_BASE (MLX5E_TC_LAST_EXPORTED_BIT + 1) 6760bd4af8SOr Gerlitz 6865ba8fb7SOr Gerlitz enum { 6960bd4af8SOr Gerlitz MLX5E_TC_FLOW_INGRESS = MLX5E_TC_INGRESS, 7060bd4af8SOr Gerlitz MLX5E_TC_FLOW_EGRESS = MLX5E_TC_EGRESS, 7160bd4af8SOr Gerlitz MLX5E_TC_FLOW_ESWITCH = BIT(MLX5E_TC_FLOW_BASE), 7260bd4af8SOr Gerlitz MLX5E_TC_FLOW_NIC = BIT(MLX5E_TC_FLOW_BASE + 1), 7360bd4af8SOr Gerlitz MLX5E_TC_FLOW_OFFLOADED = BIT(MLX5E_TC_FLOW_BASE + 2), 7460bd4af8SOr Gerlitz MLX5E_TC_FLOW_HAIRPIN = BIT(MLX5E_TC_FLOW_BASE + 3), 7560bd4af8SOr Gerlitz MLX5E_TC_FLOW_HAIRPIN_RSS = BIT(MLX5E_TC_FLOW_BASE + 4), 765dbe906fSPaul Blakey MLX5E_TC_FLOW_SLOW = BIT(MLX5E_TC_FLOW_BASE + 5), 7765ba8fb7SOr Gerlitz }; 7865ba8fb7SOr Gerlitz 79e4ad91f2SChris Mi #define MLX5E_TC_MAX_SPLITS 1 80e4ad91f2SChris Mi 8179baaec7SEli Britstein /* Helper struct for accessing a struct containing list_head array. 8279baaec7SEli Britstein * Containing struct 8379baaec7SEli Britstein * |- Helper array 8479baaec7SEli Britstein * [0] Helper item 0 8579baaec7SEli Britstein * |- list_head item 0 8679baaec7SEli Britstein * |- index (0) 8779baaec7SEli Britstein * [1] Helper item 1 8879baaec7SEli Britstein * |- list_head item 1 8979baaec7SEli Britstein * |- index (1) 9079baaec7SEli Britstein * To access the containing struct from one of the list_head items: 9179baaec7SEli Britstein * 1. Get the helper item from the list_head item using 9279baaec7SEli Britstein * helper item = 9379baaec7SEli Britstein * container_of(list_head item, helper struct type, list_head field) 9479baaec7SEli Britstein * 2. Get the contining struct from the helper item and its index in the array: 9579baaec7SEli Britstein * containing struct = 9679baaec7SEli Britstein * container_of(helper item, containing struct type, helper field[index]) 9779baaec7SEli Britstein */ 9879baaec7SEli Britstein struct encap_flow_item { 9979baaec7SEli Britstein struct list_head list; 10079baaec7SEli Britstein int index; 10179baaec7SEli Britstein }; 10279baaec7SEli Britstein 103e8f887acSAmir Vadai struct mlx5e_tc_flow { 104e8f887acSAmir Vadai struct rhash_head node; 105655dc3d2SOr Gerlitz struct mlx5e_priv *priv; 106e8f887acSAmir Vadai u64 cookie; 1075dbe906fSPaul Blakey u16 flags; 108e4ad91f2SChris Mi struct mlx5_flow_handle *rule[MLX5E_TC_MAX_SPLITS + 1]; 10979baaec7SEli Britstein /* Flow can be associated with multiple encap IDs. 11079baaec7SEli Britstein * The number of encaps is bounded by the number of supported 11179baaec7SEli Britstein * destinations. 11279baaec7SEli Britstein */ 11379baaec7SEli Britstein struct encap_flow_item encaps[MLX5_MAX_FLOW_FWD_VPORTS]; 11411c9c548SOr Gerlitz struct list_head mod_hdr; /* flows sharing the same mod hdr ID */ 1155c65c564SOr Gerlitz struct list_head hairpin; /* flows sharing the same hairpin */ 1163bc4b7bfSOr Gerlitz union { 117ecf5bb79SOr Gerlitz struct mlx5_esw_flow_attr esw_attr[0]; 1183bc4b7bfSOr Gerlitz struct mlx5_nic_flow_attr nic_attr[0]; 1193bc4b7bfSOr Gerlitz }; 120e8f887acSAmir Vadai }; 121e8f887acSAmir Vadai 12217091853SOr Gerlitz struct mlx5e_tc_flow_parse_attr { 12398b66cb1SEli Britstein struct ip_tunnel_info tun_info[MLX5_MAX_FLOW_FWD_VPORTS]; 124d11afc26SOz Shlomo struct net_device *filter_dev; 12517091853SOr Gerlitz struct mlx5_flow_spec spec; 126d79b6df6SOr Gerlitz int num_mod_hdr_actions; 127d79b6df6SOr Gerlitz void *mod_hdr_actions; 12898b66cb1SEli Britstein int mirred_ifindex[MLX5_MAX_FLOW_FWD_VPORTS]; 12917091853SOr Gerlitz }; 13017091853SOr Gerlitz 131acff797cSMaor Gottlieb #define MLX5E_TC_TABLE_NUM_GROUPS 4 132b3a433deSOr Gerlitz #define MLX5E_TC_TABLE_MAX_GROUP_SIZE BIT(16) 133e8f887acSAmir Vadai 13477ab67b7SOr Gerlitz struct mlx5e_hairpin { 13577ab67b7SOr Gerlitz struct mlx5_hairpin *pair; 13677ab67b7SOr Gerlitz 13777ab67b7SOr Gerlitz struct mlx5_core_dev *func_mdev; 1383f6d08d1SOr Gerlitz struct mlx5e_priv *func_priv; 13977ab67b7SOr Gerlitz u32 tdn; 14077ab67b7SOr Gerlitz u32 tirn; 1413f6d08d1SOr Gerlitz 1423f6d08d1SOr Gerlitz int num_channels; 1433f6d08d1SOr Gerlitz struct mlx5e_rqt indir_rqt; 1443f6d08d1SOr Gerlitz u32 indir_tirn[MLX5E_NUM_INDIR_TIRS]; 1453f6d08d1SOr Gerlitz struct mlx5e_ttc_table ttc; 14677ab67b7SOr Gerlitz }; 14777ab67b7SOr Gerlitz 1485c65c564SOr Gerlitz struct mlx5e_hairpin_entry { 1495c65c564SOr Gerlitz /* a node of a hash table which keeps all the hairpin entries */ 1505c65c564SOr Gerlitz struct hlist_node hairpin_hlist; 1515c65c564SOr Gerlitz 1525c65c564SOr Gerlitz /* flows sharing the same hairpin */ 1535c65c564SOr Gerlitz struct list_head flows; 1545c65c564SOr Gerlitz 155d8822868SOr Gerlitz u16 peer_vhca_id; 156106be53bSOr Gerlitz u8 prio; 1575c65c564SOr Gerlitz struct mlx5e_hairpin *hp; 1585c65c564SOr Gerlitz }; 1595c65c564SOr Gerlitz 16011c9c548SOr Gerlitz struct mod_hdr_key { 16111c9c548SOr Gerlitz int num_actions; 16211c9c548SOr Gerlitz void *actions; 16311c9c548SOr Gerlitz }; 16411c9c548SOr Gerlitz 16511c9c548SOr Gerlitz struct mlx5e_mod_hdr_entry { 16611c9c548SOr Gerlitz /* a node of a hash table which keeps all the mod_hdr entries */ 16711c9c548SOr Gerlitz struct hlist_node mod_hdr_hlist; 16811c9c548SOr Gerlitz 16911c9c548SOr Gerlitz /* flows sharing the same mod_hdr entry */ 17011c9c548SOr Gerlitz struct list_head flows; 17111c9c548SOr Gerlitz 17211c9c548SOr Gerlitz struct mod_hdr_key key; 17311c9c548SOr Gerlitz 17411c9c548SOr Gerlitz u32 mod_hdr_id; 17511c9c548SOr Gerlitz }; 17611c9c548SOr Gerlitz 17711c9c548SOr Gerlitz #define MLX5_MH_ACT_SZ MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto) 17811c9c548SOr Gerlitz 17911c9c548SOr Gerlitz static inline u32 hash_mod_hdr_info(struct mod_hdr_key *key) 18011c9c548SOr Gerlitz { 18111c9c548SOr Gerlitz return jhash(key->actions, 18211c9c548SOr Gerlitz key->num_actions * MLX5_MH_ACT_SZ, 0); 18311c9c548SOr Gerlitz } 18411c9c548SOr Gerlitz 18511c9c548SOr Gerlitz static inline int cmp_mod_hdr_info(struct mod_hdr_key *a, 18611c9c548SOr Gerlitz struct mod_hdr_key *b) 18711c9c548SOr Gerlitz { 18811c9c548SOr Gerlitz if (a->num_actions != b->num_actions) 18911c9c548SOr Gerlitz return 1; 19011c9c548SOr Gerlitz 19111c9c548SOr Gerlitz return memcmp(a->actions, b->actions, a->num_actions * MLX5_MH_ACT_SZ); 19211c9c548SOr Gerlitz } 19311c9c548SOr Gerlitz 19411c9c548SOr Gerlitz static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv, 19511c9c548SOr Gerlitz struct mlx5e_tc_flow *flow, 19611c9c548SOr Gerlitz struct mlx5e_tc_flow_parse_attr *parse_attr) 19711c9c548SOr Gerlitz { 19811c9c548SOr Gerlitz struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 19911c9c548SOr Gerlitz int num_actions, actions_size, namespace, err; 20011c9c548SOr Gerlitz struct mlx5e_mod_hdr_entry *mh; 20111c9c548SOr Gerlitz struct mod_hdr_key key; 20211c9c548SOr Gerlitz bool found = false; 20311c9c548SOr Gerlitz u32 hash_key; 20411c9c548SOr Gerlitz 20511c9c548SOr Gerlitz num_actions = parse_attr->num_mod_hdr_actions; 20611c9c548SOr Gerlitz actions_size = MLX5_MH_ACT_SZ * num_actions; 20711c9c548SOr Gerlitz 20811c9c548SOr Gerlitz key.actions = parse_attr->mod_hdr_actions; 20911c9c548SOr Gerlitz key.num_actions = num_actions; 21011c9c548SOr Gerlitz 21111c9c548SOr Gerlitz hash_key = hash_mod_hdr_info(&key); 21211c9c548SOr Gerlitz 21311c9c548SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_ESWITCH) { 21411c9c548SOr Gerlitz namespace = MLX5_FLOW_NAMESPACE_FDB; 21511c9c548SOr Gerlitz hash_for_each_possible(esw->offloads.mod_hdr_tbl, mh, 21611c9c548SOr Gerlitz mod_hdr_hlist, hash_key) { 21711c9c548SOr Gerlitz if (!cmp_mod_hdr_info(&mh->key, &key)) { 21811c9c548SOr Gerlitz found = true; 21911c9c548SOr Gerlitz break; 22011c9c548SOr Gerlitz } 22111c9c548SOr Gerlitz } 22211c9c548SOr Gerlitz } else { 22311c9c548SOr Gerlitz namespace = MLX5_FLOW_NAMESPACE_KERNEL; 22411c9c548SOr Gerlitz hash_for_each_possible(priv->fs.tc.mod_hdr_tbl, mh, 22511c9c548SOr Gerlitz mod_hdr_hlist, hash_key) { 22611c9c548SOr Gerlitz if (!cmp_mod_hdr_info(&mh->key, &key)) { 22711c9c548SOr Gerlitz found = true; 22811c9c548SOr Gerlitz break; 22911c9c548SOr Gerlitz } 23011c9c548SOr Gerlitz } 23111c9c548SOr Gerlitz } 23211c9c548SOr Gerlitz 23311c9c548SOr Gerlitz if (found) 23411c9c548SOr Gerlitz goto attach_flow; 23511c9c548SOr Gerlitz 23611c9c548SOr Gerlitz mh = kzalloc(sizeof(*mh) + actions_size, GFP_KERNEL); 23711c9c548SOr Gerlitz if (!mh) 23811c9c548SOr Gerlitz return -ENOMEM; 23911c9c548SOr Gerlitz 24011c9c548SOr Gerlitz mh->key.actions = (void *)mh + sizeof(*mh); 24111c9c548SOr Gerlitz memcpy(mh->key.actions, key.actions, actions_size); 24211c9c548SOr Gerlitz mh->key.num_actions = num_actions; 24311c9c548SOr Gerlitz INIT_LIST_HEAD(&mh->flows); 24411c9c548SOr Gerlitz 24511c9c548SOr Gerlitz err = mlx5_modify_header_alloc(priv->mdev, namespace, 24611c9c548SOr Gerlitz mh->key.num_actions, 24711c9c548SOr Gerlitz mh->key.actions, 24811c9c548SOr Gerlitz &mh->mod_hdr_id); 24911c9c548SOr Gerlitz if (err) 25011c9c548SOr Gerlitz goto out_err; 25111c9c548SOr Gerlitz 25211c9c548SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_ESWITCH) 25311c9c548SOr Gerlitz hash_add(esw->offloads.mod_hdr_tbl, &mh->mod_hdr_hlist, hash_key); 25411c9c548SOr Gerlitz else 25511c9c548SOr Gerlitz hash_add(priv->fs.tc.mod_hdr_tbl, &mh->mod_hdr_hlist, hash_key); 25611c9c548SOr Gerlitz 25711c9c548SOr Gerlitz attach_flow: 25811c9c548SOr Gerlitz list_add(&flow->mod_hdr, &mh->flows); 25911c9c548SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_ESWITCH) 26011c9c548SOr Gerlitz flow->esw_attr->mod_hdr_id = mh->mod_hdr_id; 26111c9c548SOr Gerlitz else 26211c9c548SOr Gerlitz flow->nic_attr->mod_hdr_id = mh->mod_hdr_id; 26311c9c548SOr Gerlitz 26411c9c548SOr Gerlitz return 0; 26511c9c548SOr Gerlitz 26611c9c548SOr Gerlitz out_err: 26711c9c548SOr Gerlitz kfree(mh); 26811c9c548SOr Gerlitz return err; 26911c9c548SOr Gerlitz } 27011c9c548SOr Gerlitz 27111c9c548SOr Gerlitz static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv, 27211c9c548SOr Gerlitz struct mlx5e_tc_flow *flow) 27311c9c548SOr Gerlitz { 27411c9c548SOr Gerlitz struct list_head *next = flow->mod_hdr.next; 27511c9c548SOr Gerlitz 27611c9c548SOr Gerlitz list_del(&flow->mod_hdr); 27711c9c548SOr Gerlitz 27811c9c548SOr Gerlitz if (list_empty(next)) { 27911c9c548SOr Gerlitz struct mlx5e_mod_hdr_entry *mh; 28011c9c548SOr Gerlitz 28111c9c548SOr Gerlitz mh = list_entry(next, struct mlx5e_mod_hdr_entry, flows); 28211c9c548SOr Gerlitz 28311c9c548SOr Gerlitz mlx5_modify_header_dealloc(priv->mdev, mh->mod_hdr_id); 28411c9c548SOr Gerlitz hash_del(&mh->mod_hdr_hlist); 28511c9c548SOr Gerlitz kfree(mh); 28611c9c548SOr Gerlitz } 28711c9c548SOr Gerlitz } 28811c9c548SOr Gerlitz 28977ab67b7SOr Gerlitz static 29077ab67b7SOr Gerlitz struct mlx5_core_dev *mlx5e_hairpin_get_mdev(struct net *net, int ifindex) 29177ab67b7SOr Gerlitz { 29277ab67b7SOr Gerlitz struct net_device *netdev; 29377ab67b7SOr Gerlitz struct mlx5e_priv *priv; 29477ab67b7SOr Gerlitz 29577ab67b7SOr Gerlitz netdev = __dev_get_by_index(net, ifindex); 29677ab67b7SOr Gerlitz priv = netdev_priv(netdev); 29777ab67b7SOr Gerlitz return priv->mdev; 29877ab67b7SOr Gerlitz } 29977ab67b7SOr Gerlitz 30077ab67b7SOr Gerlitz static int mlx5e_hairpin_create_transport(struct mlx5e_hairpin *hp) 30177ab67b7SOr Gerlitz { 30277ab67b7SOr Gerlitz u32 in[MLX5_ST_SZ_DW(create_tir_in)] = {0}; 30377ab67b7SOr Gerlitz void *tirc; 30477ab67b7SOr Gerlitz int err; 30577ab67b7SOr Gerlitz 30677ab67b7SOr Gerlitz err = mlx5_core_alloc_transport_domain(hp->func_mdev, &hp->tdn); 30777ab67b7SOr Gerlitz if (err) 30877ab67b7SOr Gerlitz goto alloc_tdn_err; 30977ab67b7SOr Gerlitz 31077ab67b7SOr Gerlitz tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 31177ab67b7SOr Gerlitz 31277ab67b7SOr Gerlitz MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT); 313ddae74acSOr Gerlitz MLX5_SET(tirc, tirc, inline_rqn, hp->pair->rqn[0]); 31477ab67b7SOr Gerlitz MLX5_SET(tirc, tirc, transport_domain, hp->tdn); 31577ab67b7SOr Gerlitz 31677ab67b7SOr Gerlitz err = mlx5_core_create_tir(hp->func_mdev, in, MLX5_ST_SZ_BYTES(create_tir_in), &hp->tirn); 31777ab67b7SOr Gerlitz if (err) 31877ab67b7SOr Gerlitz goto create_tir_err; 31977ab67b7SOr Gerlitz 32077ab67b7SOr Gerlitz return 0; 32177ab67b7SOr Gerlitz 32277ab67b7SOr Gerlitz create_tir_err: 32377ab67b7SOr Gerlitz mlx5_core_dealloc_transport_domain(hp->func_mdev, hp->tdn); 32477ab67b7SOr Gerlitz alloc_tdn_err: 32577ab67b7SOr Gerlitz return err; 32677ab67b7SOr Gerlitz } 32777ab67b7SOr Gerlitz 32877ab67b7SOr Gerlitz static void mlx5e_hairpin_destroy_transport(struct mlx5e_hairpin *hp) 32977ab67b7SOr Gerlitz { 33077ab67b7SOr Gerlitz mlx5_core_destroy_tir(hp->func_mdev, hp->tirn); 33177ab67b7SOr Gerlitz mlx5_core_dealloc_transport_domain(hp->func_mdev, hp->tdn); 33277ab67b7SOr Gerlitz } 33377ab67b7SOr Gerlitz 3343f6d08d1SOr Gerlitz static void mlx5e_hairpin_fill_rqt_rqns(struct mlx5e_hairpin *hp, void *rqtc) 3353f6d08d1SOr Gerlitz { 3363f6d08d1SOr Gerlitz u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE], rqn; 3373f6d08d1SOr Gerlitz struct mlx5e_priv *priv = hp->func_priv; 3383f6d08d1SOr Gerlitz int i, ix, sz = MLX5E_INDIR_RQT_SIZE; 3393f6d08d1SOr Gerlitz 3403f6d08d1SOr Gerlitz mlx5e_build_default_indir_rqt(indirection_rqt, sz, 3413f6d08d1SOr Gerlitz hp->num_channels); 3423f6d08d1SOr Gerlitz 3433f6d08d1SOr Gerlitz for (i = 0; i < sz; i++) { 3443f6d08d1SOr Gerlitz ix = i; 345bbeb53b8SAya Levin if (priv->rss_params.hfunc == ETH_RSS_HASH_XOR) 3463f6d08d1SOr Gerlitz ix = mlx5e_bits_invert(i, ilog2(sz)); 3473f6d08d1SOr Gerlitz ix = indirection_rqt[ix]; 3483f6d08d1SOr Gerlitz rqn = hp->pair->rqn[ix]; 3493f6d08d1SOr Gerlitz MLX5_SET(rqtc, rqtc, rq_num[i], rqn); 3503f6d08d1SOr Gerlitz } 3513f6d08d1SOr Gerlitz } 3523f6d08d1SOr Gerlitz 3533f6d08d1SOr Gerlitz static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin *hp) 3543f6d08d1SOr Gerlitz { 3553f6d08d1SOr Gerlitz int inlen, err, sz = MLX5E_INDIR_RQT_SIZE; 3563f6d08d1SOr Gerlitz struct mlx5e_priv *priv = hp->func_priv; 3573f6d08d1SOr Gerlitz struct mlx5_core_dev *mdev = priv->mdev; 3583f6d08d1SOr Gerlitz void *rqtc; 3593f6d08d1SOr Gerlitz u32 *in; 3603f6d08d1SOr Gerlitz 3613f6d08d1SOr Gerlitz inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz; 3623f6d08d1SOr Gerlitz in = kvzalloc(inlen, GFP_KERNEL); 3633f6d08d1SOr Gerlitz if (!in) 3643f6d08d1SOr Gerlitz return -ENOMEM; 3653f6d08d1SOr Gerlitz 3663f6d08d1SOr Gerlitz rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 3673f6d08d1SOr Gerlitz 3683f6d08d1SOr Gerlitz MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); 3693f6d08d1SOr Gerlitz MLX5_SET(rqtc, rqtc, rqt_max_size, sz); 3703f6d08d1SOr Gerlitz 3713f6d08d1SOr Gerlitz mlx5e_hairpin_fill_rqt_rqns(hp, rqtc); 3723f6d08d1SOr Gerlitz 3733f6d08d1SOr Gerlitz err = mlx5_core_create_rqt(mdev, in, inlen, &hp->indir_rqt.rqtn); 3743f6d08d1SOr Gerlitz if (!err) 3753f6d08d1SOr Gerlitz hp->indir_rqt.enabled = true; 3763f6d08d1SOr Gerlitz 3773f6d08d1SOr Gerlitz kvfree(in); 3783f6d08d1SOr Gerlitz return err; 3793f6d08d1SOr Gerlitz } 3803f6d08d1SOr Gerlitz 3813f6d08d1SOr Gerlitz static int mlx5e_hairpin_create_indirect_tirs(struct mlx5e_hairpin *hp) 3823f6d08d1SOr Gerlitz { 3833f6d08d1SOr Gerlitz struct mlx5e_priv *priv = hp->func_priv; 3843f6d08d1SOr Gerlitz u32 in[MLX5_ST_SZ_DW(create_tir_in)]; 3853f6d08d1SOr Gerlitz int tt, i, err; 3863f6d08d1SOr Gerlitz void *tirc; 3873f6d08d1SOr Gerlitz 3883f6d08d1SOr Gerlitz for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 389d930ac79SAya Levin struct mlx5e_tirc_config ttconfig = mlx5e_tirc_get_default_config(tt); 390d930ac79SAya Levin 3913f6d08d1SOr Gerlitz memset(in, 0, MLX5_ST_SZ_BYTES(create_tir_in)); 3923f6d08d1SOr Gerlitz tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3933f6d08d1SOr Gerlitz 3943f6d08d1SOr Gerlitz MLX5_SET(tirc, tirc, transport_domain, hp->tdn); 3953f6d08d1SOr Gerlitz MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 3963f6d08d1SOr Gerlitz MLX5_SET(tirc, tirc, indirect_table, hp->indir_rqt.rqtn); 397bbeb53b8SAya Levin mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, &ttconfig, tirc, false); 398bbeb53b8SAya Levin 3993f6d08d1SOr Gerlitz err = mlx5_core_create_tir(hp->func_mdev, in, 4003f6d08d1SOr Gerlitz MLX5_ST_SZ_BYTES(create_tir_in), &hp->indir_tirn[tt]); 4013f6d08d1SOr Gerlitz if (err) { 4023f6d08d1SOr Gerlitz mlx5_core_warn(hp->func_mdev, "create indirect tirs failed, %d\n", err); 4033f6d08d1SOr Gerlitz goto err_destroy_tirs; 4043f6d08d1SOr Gerlitz } 4053f6d08d1SOr Gerlitz } 4063f6d08d1SOr Gerlitz return 0; 4073f6d08d1SOr Gerlitz 4083f6d08d1SOr Gerlitz err_destroy_tirs: 4093f6d08d1SOr Gerlitz for (i = 0; i < tt; i++) 4103f6d08d1SOr Gerlitz mlx5_core_destroy_tir(hp->func_mdev, hp->indir_tirn[i]); 4113f6d08d1SOr Gerlitz return err; 4123f6d08d1SOr Gerlitz } 4133f6d08d1SOr Gerlitz 4143f6d08d1SOr Gerlitz static void mlx5e_hairpin_destroy_indirect_tirs(struct mlx5e_hairpin *hp) 4153f6d08d1SOr Gerlitz { 4163f6d08d1SOr Gerlitz int tt; 4173f6d08d1SOr Gerlitz 4183f6d08d1SOr Gerlitz for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) 4193f6d08d1SOr Gerlitz mlx5_core_destroy_tir(hp->func_mdev, hp->indir_tirn[tt]); 4203f6d08d1SOr Gerlitz } 4213f6d08d1SOr Gerlitz 4223f6d08d1SOr Gerlitz static void mlx5e_hairpin_set_ttc_params(struct mlx5e_hairpin *hp, 4233f6d08d1SOr Gerlitz struct ttc_params *ttc_params) 4243f6d08d1SOr Gerlitz { 4253f6d08d1SOr Gerlitz struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr; 4263f6d08d1SOr Gerlitz int tt; 4273f6d08d1SOr Gerlitz 4283f6d08d1SOr Gerlitz memset(ttc_params, 0, sizeof(*ttc_params)); 4293f6d08d1SOr Gerlitz 4303f6d08d1SOr Gerlitz ttc_params->any_tt_tirn = hp->tirn; 4313f6d08d1SOr Gerlitz 4323f6d08d1SOr Gerlitz for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) 4333f6d08d1SOr Gerlitz ttc_params->indir_tirn[tt] = hp->indir_tirn[tt]; 4343f6d08d1SOr Gerlitz 4353f6d08d1SOr Gerlitz ft_attr->max_fte = MLX5E_NUM_TT; 4363f6d08d1SOr Gerlitz ft_attr->level = MLX5E_TC_TTC_FT_LEVEL; 4373f6d08d1SOr Gerlitz ft_attr->prio = MLX5E_TC_PRIO; 4383f6d08d1SOr Gerlitz } 4393f6d08d1SOr Gerlitz 4403f6d08d1SOr Gerlitz static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp) 4413f6d08d1SOr Gerlitz { 4423f6d08d1SOr Gerlitz struct mlx5e_priv *priv = hp->func_priv; 4433f6d08d1SOr Gerlitz struct ttc_params ttc_params; 4443f6d08d1SOr Gerlitz int err; 4453f6d08d1SOr Gerlitz 4463f6d08d1SOr Gerlitz err = mlx5e_hairpin_create_indirect_rqt(hp); 4473f6d08d1SOr Gerlitz if (err) 4483f6d08d1SOr Gerlitz return err; 4493f6d08d1SOr Gerlitz 4503f6d08d1SOr Gerlitz err = mlx5e_hairpin_create_indirect_tirs(hp); 4513f6d08d1SOr Gerlitz if (err) 4523f6d08d1SOr Gerlitz goto err_create_indirect_tirs; 4533f6d08d1SOr Gerlitz 4543f6d08d1SOr Gerlitz mlx5e_hairpin_set_ttc_params(hp, &ttc_params); 4553f6d08d1SOr Gerlitz err = mlx5e_create_ttc_table(priv, &ttc_params, &hp->ttc); 4563f6d08d1SOr Gerlitz if (err) 4573f6d08d1SOr Gerlitz goto err_create_ttc_table; 4583f6d08d1SOr Gerlitz 4593f6d08d1SOr Gerlitz netdev_dbg(priv->netdev, "add hairpin: using %d channels rss ttc table id %x\n", 4603f6d08d1SOr Gerlitz hp->num_channels, hp->ttc.ft.t->id); 4613f6d08d1SOr Gerlitz 4623f6d08d1SOr Gerlitz return 0; 4633f6d08d1SOr Gerlitz 4643f6d08d1SOr Gerlitz err_create_ttc_table: 4653f6d08d1SOr Gerlitz mlx5e_hairpin_destroy_indirect_tirs(hp); 4663f6d08d1SOr Gerlitz err_create_indirect_tirs: 4673f6d08d1SOr Gerlitz mlx5e_destroy_rqt(priv, &hp->indir_rqt); 4683f6d08d1SOr Gerlitz 4693f6d08d1SOr Gerlitz return err; 4703f6d08d1SOr Gerlitz } 4713f6d08d1SOr Gerlitz 4723f6d08d1SOr Gerlitz static void mlx5e_hairpin_rss_cleanup(struct mlx5e_hairpin *hp) 4733f6d08d1SOr Gerlitz { 4743f6d08d1SOr Gerlitz struct mlx5e_priv *priv = hp->func_priv; 4753f6d08d1SOr Gerlitz 4763f6d08d1SOr Gerlitz mlx5e_destroy_ttc_table(priv, &hp->ttc); 4773f6d08d1SOr Gerlitz mlx5e_hairpin_destroy_indirect_tirs(hp); 4783f6d08d1SOr Gerlitz mlx5e_destroy_rqt(priv, &hp->indir_rqt); 4793f6d08d1SOr Gerlitz } 4803f6d08d1SOr Gerlitz 48177ab67b7SOr Gerlitz static struct mlx5e_hairpin * 48277ab67b7SOr Gerlitz mlx5e_hairpin_create(struct mlx5e_priv *priv, struct mlx5_hairpin_params *params, 48377ab67b7SOr Gerlitz int peer_ifindex) 48477ab67b7SOr Gerlitz { 48577ab67b7SOr Gerlitz struct mlx5_core_dev *func_mdev, *peer_mdev; 48677ab67b7SOr Gerlitz struct mlx5e_hairpin *hp; 48777ab67b7SOr Gerlitz struct mlx5_hairpin *pair; 48877ab67b7SOr Gerlitz int err; 48977ab67b7SOr Gerlitz 49077ab67b7SOr Gerlitz hp = kzalloc(sizeof(*hp), GFP_KERNEL); 49177ab67b7SOr Gerlitz if (!hp) 49277ab67b7SOr Gerlitz return ERR_PTR(-ENOMEM); 49377ab67b7SOr Gerlitz 49477ab67b7SOr Gerlitz func_mdev = priv->mdev; 49577ab67b7SOr Gerlitz peer_mdev = mlx5e_hairpin_get_mdev(dev_net(priv->netdev), peer_ifindex); 49677ab67b7SOr Gerlitz 49777ab67b7SOr Gerlitz pair = mlx5_core_hairpin_create(func_mdev, peer_mdev, params); 49877ab67b7SOr Gerlitz if (IS_ERR(pair)) { 49977ab67b7SOr Gerlitz err = PTR_ERR(pair); 50077ab67b7SOr Gerlitz goto create_pair_err; 50177ab67b7SOr Gerlitz } 50277ab67b7SOr Gerlitz hp->pair = pair; 50377ab67b7SOr Gerlitz hp->func_mdev = func_mdev; 5043f6d08d1SOr Gerlitz hp->func_priv = priv; 5053f6d08d1SOr Gerlitz hp->num_channels = params->num_channels; 50677ab67b7SOr Gerlitz 50777ab67b7SOr Gerlitz err = mlx5e_hairpin_create_transport(hp); 50877ab67b7SOr Gerlitz if (err) 50977ab67b7SOr Gerlitz goto create_transport_err; 51077ab67b7SOr Gerlitz 5113f6d08d1SOr Gerlitz if (hp->num_channels > 1) { 5123f6d08d1SOr Gerlitz err = mlx5e_hairpin_rss_init(hp); 5133f6d08d1SOr Gerlitz if (err) 5143f6d08d1SOr Gerlitz goto rss_init_err; 5153f6d08d1SOr Gerlitz } 5163f6d08d1SOr Gerlitz 51777ab67b7SOr Gerlitz return hp; 51877ab67b7SOr Gerlitz 5193f6d08d1SOr Gerlitz rss_init_err: 5203f6d08d1SOr Gerlitz mlx5e_hairpin_destroy_transport(hp); 52177ab67b7SOr Gerlitz create_transport_err: 52277ab67b7SOr Gerlitz mlx5_core_hairpin_destroy(hp->pair); 52377ab67b7SOr Gerlitz create_pair_err: 52477ab67b7SOr Gerlitz kfree(hp); 52577ab67b7SOr Gerlitz return ERR_PTR(err); 52677ab67b7SOr Gerlitz } 52777ab67b7SOr Gerlitz 52877ab67b7SOr Gerlitz static void mlx5e_hairpin_destroy(struct mlx5e_hairpin *hp) 52977ab67b7SOr Gerlitz { 5303f6d08d1SOr Gerlitz if (hp->num_channels > 1) 5313f6d08d1SOr Gerlitz mlx5e_hairpin_rss_cleanup(hp); 53277ab67b7SOr Gerlitz mlx5e_hairpin_destroy_transport(hp); 53377ab67b7SOr Gerlitz mlx5_core_hairpin_destroy(hp->pair); 53477ab67b7SOr Gerlitz kvfree(hp); 53577ab67b7SOr Gerlitz } 53677ab67b7SOr Gerlitz 537106be53bSOr Gerlitz static inline u32 hash_hairpin_info(u16 peer_vhca_id, u8 prio) 538106be53bSOr Gerlitz { 539106be53bSOr Gerlitz return (peer_vhca_id << 16 | prio); 540106be53bSOr Gerlitz } 541106be53bSOr Gerlitz 5425c65c564SOr Gerlitz static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv, 543106be53bSOr Gerlitz u16 peer_vhca_id, u8 prio) 5445c65c564SOr Gerlitz { 5455c65c564SOr Gerlitz struct mlx5e_hairpin_entry *hpe; 546106be53bSOr Gerlitz u32 hash_key = hash_hairpin_info(peer_vhca_id, prio); 5475c65c564SOr Gerlitz 5485c65c564SOr Gerlitz hash_for_each_possible(priv->fs.tc.hairpin_tbl, hpe, 549106be53bSOr Gerlitz hairpin_hlist, hash_key) { 550106be53bSOr Gerlitz if (hpe->peer_vhca_id == peer_vhca_id && hpe->prio == prio) 5515c65c564SOr Gerlitz return hpe; 5525c65c564SOr Gerlitz } 5535c65c564SOr Gerlitz 5545c65c564SOr Gerlitz return NULL; 5555c65c564SOr Gerlitz } 5565c65c564SOr Gerlitz 557106be53bSOr Gerlitz #define UNKNOWN_MATCH_PRIO 8 558106be53bSOr Gerlitz 559106be53bSOr Gerlitz static int mlx5e_hairpin_get_prio(struct mlx5e_priv *priv, 560e98bedf5SEli Britstein struct mlx5_flow_spec *spec, u8 *match_prio, 561e98bedf5SEli Britstein struct netlink_ext_ack *extack) 562106be53bSOr Gerlitz { 563106be53bSOr Gerlitz void *headers_c, *headers_v; 564106be53bSOr Gerlitz u8 prio_val, prio_mask = 0; 565106be53bSOr Gerlitz bool vlan_present; 566106be53bSOr Gerlitz 567106be53bSOr Gerlitz #ifdef CONFIG_MLX5_CORE_EN_DCB 568106be53bSOr Gerlitz if (priv->dcbx_dp.trust_state != MLX5_QPTS_TRUST_PCP) { 569e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 570e98bedf5SEli Britstein "only PCP trust state supported for hairpin"); 571106be53bSOr Gerlitz return -EOPNOTSUPP; 572106be53bSOr Gerlitz } 573106be53bSOr Gerlitz #endif 574106be53bSOr Gerlitz headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); 575106be53bSOr Gerlitz headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 576106be53bSOr Gerlitz 577106be53bSOr Gerlitz vlan_present = MLX5_GET(fte_match_set_lyr_2_4, headers_v, cvlan_tag); 578106be53bSOr Gerlitz if (vlan_present) { 579106be53bSOr Gerlitz prio_mask = MLX5_GET(fte_match_set_lyr_2_4, headers_c, first_prio); 580106be53bSOr Gerlitz prio_val = MLX5_GET(fte_match_set_lyr_2_4, headers_v, first_prio); 581106be53bSOr Gerlitz } 582106be53bSOr Gerlitz 583106be53bSOr Gerlitz if (!vlan_present || !prio_mask) { 584106be53bSOr Gerlitz prio_val = UNKNOWN_MATCH_PRIO; 585106be53bSOr Gerlitz } else if (prio_mask != 0x7) { 586e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 587e98bedf5SEli Britstein "masked priority match not supported for hairpin"); 588106be53bSOr Gerlitz return -EOPNOTSUPP; 589106be53bSOr Gerlitz } 590106be53bSOr Gerlitz 591106be53bSOr Gerlitz *match_prio = prio_val; 592106be53bSOr Gerlitz return 0; 593106be53bSOr Gerlitz } 594106be53bSOr Gerlitz 5955c65c564SOr Gerlitz static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, 5965c65c564SOr Gerlitz struct mlx5e_tc_flow *flow, 597e98bedf5SEli Britstein struct mlx5e_tc_flow_parse_attr *parse_attr, 598e98bedf5SEli Britstein struct netlink_ext_ack *extack) 5995c65c564SOr Gerlitz { 60098b66cb1SEli Britstein int peer_ifindex = parse_attr->mirred_ifindex[0]; 6015c65c564SOr Gerlitz struct mlx5_hairpin_params params; 602d8822868SOr Gerlitz struct mlx5_core_dev *peer_mdev; 6035c65c564SOr Gerlitz struct mlx5e_hairpin_entry *hpe; 6045c65c564SOr Gerlitz struct mlx5e_hairpin *hp; 6053f6d08d1SOr Gerlitz u64 link_speed64; 6063f6d08d1SOr Gerlitz u32 link_speed; 607106be53bSOr Gerlitz u8 match_prio; 608d8822868SOr Gerlitz u16 peer_id; 6095c65c564SOr Gerlitz int err; 6105c65c564SOr Gerlitz 611d8822868SOr Gerlitz peer_mdev = mlx5e_hairpin_get_mdev(dev_net(priv->netdev), peer_ifindex); 612d8822868SOr Gerlitz if (!MLX5_CAP_GEN(priv->mdev, hairpin) || !MLX5_CAP_GEN(peer_mdev, hairpin)) { 613e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, "hairpin is not supported"); 6145c65c564SOr Gerlitz return -EOPNOTSUPP; 6155c65c564SOr Gerlitz } 6165c65c564SOr Gerlitz 617d8822868SOr Gerlitz peer_id = MLX5_CAP_GEN(peer_mdev, vhca_id); 618e98bedf5SEli Britstein err = mlx5e_hairpin_get_prio(priv, &parse_attr->spec, &match_prio, 619e98bedf5SEli Britstein extack); 620106be53bSOr Gerlitz if (err) 621106be53bSOr Gerlitz return err; 622106be53bSOr Gerlitz hpe = mlx5e_hairpin_get(priv, peer_id, match_prio); 6235c65c564SOr Gerlitz if (hpe) 6245c65c564SOr Gerlitz goto attach_flow; 6255c65c564SOr Gerlitz 6265c65c564SOr Gerlitz hpe = kzalloc(sizeof(*hpe), GFP_KERNEL); 6275c65c564SOr Gerlitz if (!hpe) 6285c65c564SOr Gerlitz return -ENOMEM; 6295c65c564SOr Gerlitz 6305c65c564SOr Gerlitz INIT_LIST_HEAD(&hpe->flows); 631d8822868SOr Gerlitz hpe->peer_vhca_id = peer_id; 632106be53bSOr Gerlitz hpe->prio = match_prio; 6335c65c564SOr Gerlitz 6345c65c564SOr Gerlitz params.log_data_size = 15; 6355c65c564SOr Gerlitz params.log_data_size = min_t(u8, params.log_data_size, 6365c65c564SOr Gerlitz MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz)); 6375c65c564SOr Gerlitz params.log_data_size = max_t(u8, params.log_data_size, 6385c65c564SOr Gerlitz MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz)); 6395c65c564SOr Gerlitz 640eb9180f7SOr Gerlitz params.log_num_packets = params.log_data_size - 641eb9180f7SOr Gerlitz MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(priv->mdev); 642eb9180f7SOr Gerlitz params.log_num_packets = min_t(u8, params.log_num_packets, 643eb9180f7SOr Gerlitz MLX5_CAP_GEN(priv->mdev, log_max_hairpin_num_packets)); 644eb9180f7SOr Gerlitz 645eb9180f7SOr Gerlitz params.q_counter = priv->q_counter; 6463f6d08d1SOr Gerlitz /* set hairpin pair per each 50Gbs share of the link */ 6472c81bfd5SHuy Nguyen mlx5e_port_max_linkspeed(priv->mdev, &link_speed); 6483f6d08d1SOr Gerlitz link_speed = max_t(u32, link_speed, 50000); 6493f6d08d1SOr Gerlitz link_speed64 = link_speed; 6503f6d08d1SOr Gerlitz do_div(link_speed64, 50000); 6513f6d08d1SOr Gerlitz params.num_channels = link_speed64; 6523f6d08d1SOr Gerlitz 6535c65c564SOr Gerlitz hp = mlx5e_hairpin_create(priv, ¶ms, peer_ifindex); 6545c65c564SOr Gerlitz if (IS_ERR(hp)) { 6555c65c564SOr Gerlitz err = PTR_ERR(hp); 6565c65c564SOr Gerlitz goto create_hairpin_err; 6575c65c564SOr Gerlitz } 6585c65c564SOr Gerlitz 659eb9180f7SOr Gerlitz netdev_dbg(priv->netdev, "add hairpin: tirn %x rqn %x peer %s sqn %x prio %d (log) data %d packets %d\n", 660ddae74acSOr Gerlitz hp->tirn, hp->pair->rqn[0], hp->pair->peer_mdev->priv.name, 661eb9180f7SOr Gerlitz hp->pair->sqn[0], match_prio, params.log_data_size, params.log_num_packets); 6625c65c564SOr Gerlitz 6635c65c564SOr Gerlitz hpe->hp = hp; 664106be53bSOr Gerlitz hash_add(priv->fs.tc.hairpin_tbl, &hpe->hairpin_hlist, 665106be53bSOr Gerlitz hash_hairpin_info(peer_id, match_prio)); 6665c65c564SOr Gerlitz 6675c65c564SOr Gerlitz attach_flow: 6683f6d08d1SOr Gerlitz if (hpe->hp->num_channels > 1) { 6693f6d08d1SOr Gerlitz flow->flags |= MLX5E_TC_FLOW_HAIRPIN_RSS; 6703f6d08d1SOr Gerlitz flow->nic_attr->hairpin_ft = hpe->hp->ttc.ft.t; 6713f6d08d1SOr Gerlitz } else { 6725c65c564SOr Gerlitz flow->nic_attr->hairpin_tirn = hpe->hp->tirn; 6733f6d08d1SOr Gerlitz } 6745c65c564SOr Gerlitz list_add(&flow->hairpin, &hpe->flows); 6753f6d08d1SOr Gerlitz 6765c65c564SOr Gerlitz return 0; 6775c65c564SOr Gerlitz 6785c65c564SOr Gerlitz create_hairpin_err: 6795c65c564SOr Gerlitz kfree(hpe); 6805c65c564SOr Gerlitz return err; 6815c65c564SOr Gerlitz } 6825c65c564SOr Gerlitz 6835c65c564SOr Gerlitz static void mlx5e_hairpin_flow_del(struct mlx5e_priv *priv, 6845c65c564SOr Gerlitz struct mlx5e_tc_flow *flow) 6855c65c564SOr Gerlitz { 6865c65c564SOr Gerlitz struct list_head *next = flow->hairpin.next; 6875c65c564SOr Gerlitz 6885c65c564SOr Gerlitz list_del(&flow->hairpin); 6895c65c564SOr Gerlitz 6905c65c564SOr Gerlitz /* no more hairpin flows for us, release the hairpin pair */ 6915c65c564SOr Gerlitz if (list_empty(next)) { 6925c65c564SOr Gerlitz struct mlx5e_hairpin_entry *hpe; 6935c65c564SOr Gerlitz 6945c65c564SOr Gerlitz hpe = list_entry(next, struct mlx5e_hairpin_entry, flows); 6955c65c564SOr Gerlitz 6965c65c564SOr Gerlitz netdev_dbg(priv->netdev, "del hairpin: peer %s\n", 6975c65c564SOr Gerlitz hpe->hp->pair->peer_mdev->priv.name); 6985c65c564SOr Gerlitz 6995c65c564SOr Gerlitz mlx5e_hairpin_destroy(hpe->hp); 7005c65c564SOr Gerlitz hash_del(&hpe->hairpin_hlist); 7015c65c564SOr Gerlitz kfree(hpe); 7025c65c564SOr Gerlitz } 7035c65c564SOr Gerlitz } 7045c65c564SOr Gerlitz 705c83954abSRabie Loulou static int 70674491de9SMark Bloch mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv, 70717091853SOr Gerlitz struct mlx5e_tc_flow_parse_attr *parse_attr, 708e98bedf5SEli Britstein struct mlx5e_tc_flow *flow, 709e98bedf5SEli Britstein struct netlink_ext_ack *extack) 710e8f887acSAmir Vadai { 711aa0cbbaeSOr Gerlitz struct mlx5_nic_flow_attr *attr = flow->nic_attr; 712aad7e08dSAmir Vadai struct mlx5_core_dev *dev = priv->mdev; 7135c65c564SOr Gerlitz struct mlx5_flow_destination dest[2] = {}; 71466958ed9SHadar Hen Zion struct mlx5_flow_act flow_act = { 7153bc4b7bfSOr Gerlitz .action = attr->action, 7163bc4b7bfSOr Gerlitz .flow_tag = attr->flow_tag, 71760786f09SMark Bloch .reformat_id = 0, 71842f7ad67SPaul Blakey .flags = FLOW_ACT_HAS_TAG | FLOW_ACT_NO_APPEND, 71966958ed9SHadar Hen Zion }; 720aad7e08dSAmir Vadai struct mlx5_fc *counter = NULL; 721e8f887acSAmir Vadai bool table_created = false; 7225c65c564SOr Gerlitz int err, dest_ix = 0; 723e8f887acSAmir Vadai 7245c65c564SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_HAIRPIN) { 725e98bedf5SEli Britstein err = mlx5e_hairpin_flow_add(priv, flow, parse_attr, extack); 7265c65c564SOr Gerlitz if (err) { 7275c65c564SOr Gerlitz goto err_add_hairpin_flow; 7285c65c564SOr Gerlitz } 7293f6d08d1SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_HAIRPIN_RSS) { 7303f6d08d1SOr Gerlitz dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 7313f6d08d1SOr Gerlitz dest[dest_ix].ft = attr->hairpin_ft; 7323f6d08d1SOr Gerlitz } else { 7335c65c564SOr Gerlitz dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 7345c65c564SOr Gerlitz dest[dest_ix].tir_num = attr->hairpin_tirn; 7353f6d08d1SOr Gerlitz } 7363f6d08d1SOr Gerlitz dest_ix++; 7373f6d08d1SOr Gerlitz } else if (attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 7385c65c564SOr Gerlitz dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 7395c65c564SOr Gerlitz dest[dest_ix].ft = priv->fs.vlan.ft.t; 7405c65c564SOr Gerlitz dest_ix++; 7415c65c564SOr Gerlitz } 742aad7e08dSAmir Vadai 7435c65c564SOr Gerlitz if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 7445c65c564SOr Gerlitz counter = mlx5_fc_create(dev, true); 7455c65c564SOr Gerlitz if (IS_ERR(counter)) { 746c83954abSRabie Loulou err = PTR_ERR(counter); 7475c65c564SOr Gerlitz goto err_fc_create; 7485c65c564SOr Gerlitz } 7495c65c564SOr Gerlitz dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 750171c7625SMark Bloch dest[dest_ix].counter_id = mlx5_fc_id(counter); 7515c65c564SOr Gerlitz dest_ix++; 752b8aee822SMark Bloch attr->counter = counter; 753aad7e08dSAmir Vadai } 754aad7e08dSAmir Vadai 7552f4fe4caSOr Gerlitz if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { 7563099eb5aSOr Gerlitz err = mlx5e_attach_mod_hdr(priv, flow, parse_attr); 757d7e75a32SOr Gerlitz flow_act.modify_id = attr->mod_hdr_id; 7582f4fe4caSOr Gerlitz kfree(parse_attr->mod_hdr_actions); 759c83954abSRabie Loulou if (err) 7602f4fe4caSOr Gerlitz goto err_create_mod_hdr_id; 7612f4fe4caSOr Gerlitz } 7622f4fe4caSOr Gerlitz 763acff797cSMaor Gottlieb if (IS_ERR_OR_NULL(priv->fs.tc.t)) { 76421b9c144SOr Gerlitz int tc_grp_size, tc_tbl_size; 76521b9c144SOr Gerlitz u32 max_flow_counter; 76621b9c144SOr Gerlitz 76721b9c144SOr Gerlitz max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) | 76821b9c144SOr Gerlitz MLX5_CAP_GEN(dev, max_flow_counter_15_0); 76921b9c144SOr Gerlitz 77021b9c144SOr Gerlitz tc_grp_size = min_t(int, max_flow_counter, MLX5E_TC_TABLE_MAX_GROUP_SIZE); 77121b9c144SOr Gerlitz 77221b9c144SOr Gerlitz tc_tbl_size = min_t(int, tc_grp_size * MLX5E_TC_TABLE_NUM_GROUPS, 77321b9c144SOr Gerlitz BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev, log_max_ft_size))); 77421b9c144SOr Gerlitz 775acff797cSMaor Gottlieb priv->fs.tc.t = 776acff797cSMaor Gottlieb mlx5_create_auto_grouped_flow_table(priv->fs.ns, 777acff797cSMaor Gottlieb MLX5E_TC_PRIO, 77821b9c144SOr Gerlitz tc_tbl_size, 779acff797cSMaor Gottlieb MLX5E_TC_TABLE_NUM_GROUPS, 7803f6d08d1SOr Gerlitz MLX5E_TC_FT_LEVEL, 0); 781acff797cSMaor Gottlieb if (IS_ERR(priv->fs.tc.t)) { 782e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 783e98bedf5SEli Britstein "Failed to create tc offload table\n"); 784e8f887acSAmir Vadai netdev_err(priv->netdev, 785e8f887acSAmir Vadai "Failed to create tc offload table\n"); 786c83954abSRabie Loulou err = PTR_ERR(priv->fs.tc.t); 787aad7e08dSAmir Vadai goto err_create_ft; 788e8f887acSAmir Vadai } 789e8f887acSAmir Vadai 790e8f887acSAmir Vadai table_created = true; 791e8f887acSAmir Vadai } 792e8f887acSAmir Vadai 79338aa51c1SOr Gerlitz if (attr->match_level != MLX5_MATCH_NONE) 79417091853SOr Gerlitz parse_attr->spec.match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 79538aa51c1SOr Gerlitz 796c83954abSRabie Loulou flow->rule[0] = mlx5_add_flow_rules(priv->fs.tc.t, &parse_attr->spec, 7975c65c564SOr Gerlitz &flow_act, dest, dest_ix); 798e8f887acSAmir Vadai 799c83954abSRabie Loulou if (IS_ERR(flow->rule[0])) { 800c83954abSRabie Loulou err = PTR_ERR(flow->rule[0]); 801aad7e08dSAmir Vadai goto err_add_rule; 802c83954abSRabie Loulou } 803aad7e08dSAmir Vadai 804c83954abSRabie Loulou return 0; 805aad7e08dSAmir Vadai 806aad7e08dSAmir Vadai err_add_rule: 807aad7e08dSAmir Vadai if (table_created) { 808acff797cSMaor Gottlieb mlx5_destroy_flow_table(priv->fs.tc.t); 809acff797cSMaor Gottlieb priv->fs.tc.t = NULL; 810e8f887acSAmir Vadai } 811aad7e08dSAmir Vadai err_create_ft: 8122f4fe4caSOr Gerlitz if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 8133099eb5aSOr Gerlitz mlx5e_detach_mod_hdr(priv, flow); 8142f4fe4caSOr Gerlitz err_create_mod_hdr_id: 815aad7e08dSAmir Vadai mlx5_fc_destroy(dev, counter); 8165c65c564SOr Gerlitz err_fc_create: 8175c65c564SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_HAIRPIN) 8185c65c564SOr Gerlitz mlx5e_hairpin_flow_del(priv, flow); 8195c65c564SOr Gerlitz err_add_hairpin_flow: 820c83954abSRabie Loulou return err; 821e8f887acSAmir Vadai } 822e8f887acSAmir Vadai 823d85cdccbSOr Gerlitz static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv, 824d85cdccbSOr Gerlitz struct mlx5e_tc_flow *flow) 825d85cdccbSOr Gerlitz { 826513f8f7fSOr Gerlitz struct mlx5_nic_flow_attr *attr = flow->nic_attr; 827d85cdccbSOr Gerlitz struct mlx5_fc *counter = NULL; 828d85cdccbSOr Gerlitz 829b8aee822SMark Bloch counter = attr->counter; 830e4ad91f2SChris Mi mlx5_del_flow_rules(flow->rule[0]); 831d85cdccbSOr Gerlitz mlx5_fc_destroy(priv->mdev, counter); 832d85cdccbSOr Gerlitz 833b3a433deSOr Gerlitz if (!mlx5e_tc_num_filters(priv) && priv->fs.tc.t) { 834d85cdccbSOr Gerlitz mlx5_destroy_flow_table(priv->fs.tc.t); 835d85cdccbSOr Gerlitz priv->fs.tc.t = NULL; 836d85cdccbSOr Gerlitz } 8372f4fe4caSOr Gerlitz 838513f8f7fSOr Gerlitz if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 8393099eb5aSOr Gerlitz mlx5e_detach_mod_hdr(priv, flow); 8405c65c564SOr Gerlitz 8415c65c564SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_HAIRPIN) 8425c65c564SOr Gerlitz mlx5e_hairpin_flow_del(priv, flow); 843d85cdccbSOr Gerlitz } 844d85cdccbSOr Gerlitz 845aa0cbbaeSOr Gerlitz static void mlx5e_detach_encap(struct mlx5e_priv *priv, 846aa0cbbaeSOr Gerlitz struct mlx5e_tc_flow *flow); 847aa0cbbaeSOr Gerlitz 8483c37745eSOr Gerlitz static int mlx5e_attach_encap(struct mlx5e_priv *priv, 8493c37745eSOr Gerlitz struct ip_tunnel_info *tun_info, 8503c37745eSOr Gerlitz struct net_device *mirred_dev, 8513c37745eSOr Gerlitz struct net_device **encap_dev, 852e98bedf5SEli Britstein struct mlx5e_tc_flow *flow, 853e98bedf5SEli Britstein struct netlink_ext_ack *extack); 8543c37745eSOr Gerlitz 8556d2a3ed0SOr Gerlitz static struct mlx5_flow_handle * 8566d2a3ed0SOr Gerlitz mlx5e_tc_offload_fdb_rules(struct mlx5_eswitch *esw, 8576d2a3ed0SOr Gerlitz struct mlx5e_tc_flow *flow, 8586d2a3ed0SOr Gerlitz struct mlx5_flow_spec *spec, 8596d2a3ed0SOr Gerlitz struct mlx5_esw_flow_attr *attr) 8606d2a3ed0SOr Gerlitz { 8616d2a3ed0SOr Gerlitz struct mlx5_flow_handle *rule; 8626d2a3ed0SOr Gerlitz 8636d2a3ed0SOr Gerlitz rule = mlx5_eswitch_add_offloaded_rule(esw, spec, attr); 8646d2a3ed0SOr Gerlitz if (IS_ERR(rule)) 8656d2a3ed0SOr Gerlitz return rule; 8666d2a3ed0SOr Gerlitz 867e85e02baSEli Britstein if (attr->split_count) { 8686d2a3ed0SOr Gerlitz flow->rule[1] = mlx5_eswitch_add_fwd_rule(esw, spec, attr); 8696d2a3ed0SOr Gerlitz if (IS_ERR(flow->rule[1])) { 8706d2a3ed0SOr Gerlitz mlx5_eswitch_del_offloaded_rule(esw, rule, attr); 8716d2a3ed0SOr Gerlitz return flow->rule[1]; 8726d2a3ed0SOr Gerlitz } 8736d2a3ed0SOr Gerlitz } 8746d2a3ed0SOr Gerlitz 8756d2a3ed0SOr Gerlitz flow->flags |= MLX5E_TC_FLOW_OFFLOADED; 8766d2a3ed0SOr Gerlitz return rule; 8776d2a3ed0SOr Gerlitz } 8786d2a3ed0SOr Gerlitz 8796d2a3ed0SOr Gerlitz static void 8806d2a3ed0SOr Gerlitz mlx5e_tc_unoffload_fdb_rules(struct mlx5_eswitch *esw, 8816d2a3ed0SOr Gerlitz struct mlx5e_tc_flow *flow, 8826d2a3ed0SOr Gerlitz struct mlx5_esw_flow_attr *attr) 8836d2a3ed0SOr Gerlitz { 8846d2a3ed0SOr Gerlitz flow->flags &= ~MLX5E_TC_FLOW_OFFLOADED; 8856d2a3ed0SOr Gerlitz 886e85e02baSEli Britstein if (attr->split_count) 8876d2a3ed0SOr Gerlitz mlx5_eswitch_del_fwd_rule(esw, flow->rule[1], attr); 8886d2a3ed0SOr Gerlitz 8896d2a3ed0SOr Gerlitz mlx5_eswitch_del_offloaded_rule(esw, flow->rule[0], attr); 8906d2a3ed0SOr Gerlitz } 8916d2a3ed0SOr Gerlitz 8925dbe906fSPaul Blakey static struct mlx5_flow_handle * 8935dbe906fSPaul Blakey mlx5e_tc_offload_to_slow_path(struct mlx5_eswitch *esw, 8945dbe906fSPaul Blakey struct mlx5e_tc_flow *flow, 8955dbe906fSPaul Blakey struct mlx5_flow_spec *spec, 8965dbe906fSPaul Blakey struct mlx5_esw_flow_attr *slow_attr) 8975dbe906fSPaul Blakey { 8985dbe906fSPaul Blakey struct mlx5_flow_handle *rule; 8995dbe906fSPaul Blakey 9005dbe906fSPaul Blakey memcpy(slow_attr, flow->esw_attr, sizeof(*slow_attr)); 9015dbe906fSPaul Blakey slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, 902e85e02baSEli Britstein slow_attr->split_count = 0, 9035dbe906fSPaul Blakey slow_attr->dest_chain = FDB_SLOW_PATH_CHAIN, 9045dbe906fSPaul Blakey 9055dbe906fSPaul Blakey rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, slow_attr); 9065dbe906fSPaul Blakey if (!IS_ERR(rule)) 9075dbe906fSPaul Blakey flow->flags |= MLX5E_TC_FLOW_SLOW; 9085dbe906fSPaul Blakey 9095dbe906fSPaul Blakey return rule; 9105dbe906fSPaul Blakey } 9115dbe906fSPaul Blakey 9125dbe906fSPaul Blakey static void 9135dbe906fSPaul Blakey mlx5e_tc_unoffload_from_slow_path(struct mlx5_eswitch *esw, 9145dbe906fSPaul Blakey struct mlx5e_tc_flow *flow, 9155dbe906fSPaul Blakey struct mlx5_esw_flow_attr *slow_attr) 9165dbe906fSPaul Blakey { 9175dbe906fSPaul Blakey memcpy(slow_attr, flow->esw_attr, sizeof(*slow_attr)); 9185dbe906fSPaul Blakey mlx5e_tc_unoffload_fdb_rules(esw, flow, slow_attr); 9195dbe906fSPaul Blakey flow->flags &= ~MLX5E_TC_FLOW_SLOW; 9205dbe906fSPaul Blakey } 9215dbe906fSPaul Blakey 922c83954abSRabie Loulou static int 92374491de9SMark Bloch mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, 92417091853SOr Gerlitz struct mlx5e_tc_flow_parse_attr *parse_attr, 925e98bedf5SEli Britstein struct mlx5e_tc_flow *flow, 926e98bedf5SEli Britstein struct netlink_ext_ack *extack) 927adb4c123SOr Gerlitz { 928adb4c123SOr Gerlitz struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 929bf07aa73SPaul Blakey u32 max_chain = mlx5_eswitch_get_chain_range(esw); 930aa0cbbaeSOr Gerlitz struct mlx5_esw_flow_attr *attr = flow->esw_attr; 931bf07aa73SPaul Blakey u16 max_prio = mlx5_eswitch_get_prio_range(esw); 9323c37745eSOr Gerlitz struct net_device *out_dev, *encap_dev = NULL; 933b8aee822SMark Bloch struct mlx5_fc *counter = NULL; 9343c37745eSOr Gerlitz struct mlx5e_rep_priv *rpriv; 9353c37745eSOr Gerlitz struct mlx5e_priv *out_priv; 936c83954abSRabie Loulou int err = 0, encap_err = 0; 937f493f155SEli Britstein int out_index; 9388b32580dSOr Gerlitz 939bf07aa73SPaul Blakey /* if prios are not supported, keep the old behaviour of using same prio 940bf07aa73SPaul Blakey * for all offloaded rules. 941bf07aa73SPaul Blakey */ 942bf07aa73SPaul Blakey if (!mlx5_eswitch_prios_supported(esw)) 943e52c2802SPaul Blakey attr->prio = 1; 944e52c2802SPaul Blakey 945bf07aa73SPaul Blakey if (attr->chain > max_chain) { 946bf07aa73SPaul Blakey NL_SET_ERR_MSG(extack, "Requested chain is out of supported range"); 947bf07aa73SPaul Blakey err = -EOPNOTSUPP; 948bf07aa73SPaul Blakey goto err_max_prio_chain; 949bf07aa73SPaul Blakey } 950bf07aa73SPaul Blakey 951bf07aa73SPaul Blakey if (attr->prio > max_prio) { 952bf07aa73SPaul Blakey NL_SET_ERR_MSG(extack, "Requested priority is out of supported range"); 953bf07aa73SPaul Blakey err = -EOPNOTSUPP; 954bf07aa73SPaul Blakey goto err_max_prio_chain; 955bf07aa73SPaul Blakey } 956bf07aa73SPaul Blakey 957f493f155SEli Britstein for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) { 958f493f155SEli Britstein if (!(attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP)) 959f493f155SEli Britstein continue; 960f493f155SEli Britstein 9613c37745eSOr Gerlitz out_dev = __dev_get_by_index(dev_net(priv->netdev), 96298b66cb1SEli Britstein attr->parse_attr->mirred_ifindex[0]); 96398b66cb1SEli Britstein encap_err = mlx5e_attach_encap(priv, &parse_attr->tun_info[0], 964c83954abSRabie Loulou out_dev, &encap_dev, flow, 965c83954abSRabie Loulou extack); 966c83954abSRabie Loulou if (encap_err && encap_err != -EAGAIN) { 967c83954abSRabie Loulou err = encap_err; 9683c37745eSOr Gerlitz goto err_attach_encap; 9693c37745eSOr Gerlitz } 9703c37745eSOr Gerlitz out_priv = netdev_priv(encap_dev); 9713c37745eSOr Gerlitz rpriv = out_priv->ppriv; 9721cc26d74SEli Britstein attr->dests[out_index].rep = rpriv->rep; 9731cc26d74SEli Britstein attr->dests[out_index].mdev = out_priv->mdev; 9743c37745eSOr Gerlitz } 9753c37745eSOr Gerlitz 9768b32580dSOr Gerlitz err = mlx5_eswitch_add_vlan_action(esw, attr); 977c83954abSRabie Loulou if (err) 978aa0cbbaeSOr Gerlitz goto err_add_vlan; 979adb4c123SOr Gerlitz 980d7e75a32SOr Gerlitz if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { 9811a9527bbSOr Gerlitz err = mlx5e_attach_mod_hdr(priv, flow, parse_attr); 982d7e75a32SOr Gerlitz kfree(parse_attr->mod_hdr_actions); 983c83954abSRabie Loulou if (err) 984d7e75a32SOr Gerlitz goto err_mod_hdr; 985d7e75a32SOr Gerlitz } 986d7e75a32SOr Gerlitz 987b8aee822SMark Bloch if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 988b8aee822SMark Bloch counter = mlx5_fc_create(esw->dev, true); 989b8aee822SMark Bloch if (IS_ERR(counter)) { 990c83954abSRabie Loulou err = PTR_ERR(counter); 991b8aee822SMark Bloch goto err_create_counter; 992b8aee822SMark Bloch } 993b8aee822SMark Bloch 994b8aee822SMark Bloch attr->counter = counter; 995b8aee822SMark Bloch } 996b8aee822SMark Bloch 997c83954abSRabie Loulou /* we get here if (1) there's no error or when 9983c37745eSOr Gerlitz * (2) there's an encap action and we're on -EAGAIN (no valid neigh) 9993c37745eSOr Gerlitz */ 10005dbe906fSPaul Blakey if (encap_err == -EAGAIN) { 10015dbe906fSPaul Blakey /* continue with goto slow path rule instead */ 10025dbe906fSPaul Blakey struct mlx5_esw_flow_attr slow_attr; 10035dbe906fSPaul Blakey 10045dbe906fSPaul Blakey flow->rule[0] = mlx5e_tc_offload_to_slow_path(esw, flow, &parse_attr->spec, &slow_attr); 10055dbe906fSPaul Blakey } else { 10066d2a3ed0SOr Gerlitz flow->rule[0] = mlx5e_tc_offload_fdb_rules(esw, flow, &parse_attr->spec, attr); 10075dbe906fSPaul Blakey } 10085dbe906fSPaul Blakey 1009c83954abSRabie Loulou if (IS_ERR(flow->rule[0])) { 1010c83954abSRabie Loulou err = PTR_ERR(flow->rule[0]); 1011aa0cbbaeSOr Gerlitz goto err_add_rule; 1012c83954abSRabie Loulou } 1013c83954abSRabie Loulou 10145dbe906fSPaul Blakey return 0; 1015aa0cbbaeSOr Gerlitz 1016aa0cbbaeSOr Gerlitz err_add_rule: 1017b8aee822SMark Bloch mlx5_fc_destroy(esw->dev, counter); 1018b8aee822SMark Bloch err_create_counter: 1019513f8f7fSOr Gerlitz if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 10201a9527bbSOr Gerlitz mlx5e_detach_mod_hdr(priv, flow); 1021d7e75a32SOr Gerlitz err_mod_hdr: 1022aa0cbbaeSOr Gerlitz mlx5_eswitch_del_vlan_action(esw, attr); 1023aa0cbbaeSOr Gerlitz err_add_vlan: 1024f493f155SEli Britstein for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) 1025f493f155SEli Britstein if (attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP) { 1026aa0cbbaeSOr Gerlitz mlx5e_detach_encap(priv, flow); 1027f493f155SEli Britstein break; 1028f493f155SEli Britstein } 10293c37745eSOr Gerlitz err_attach_encap: 1030bf07aa73SPaul Blakey err_max_prio_chain: 1031c83954abSRabie Loulou return err; 1032aa0cbbaeSOr Gerlitz } 1033d85cdccbSOr Gerlitz 1034d85cdccbSOr Gerlitz static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, 1035d85cdccbSOr Gerlitz struct mlx5e_tc_flow *flow) 1036d85cdccbSOr Gerlitz { 1037d85cdccbSOr Gerlitz struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1038d7e75a32SOr Gerlitz struct mlx5_esw_flow_attr *attr = flow->esw_attr; 10395dbe906fSPaul Blakey struct mlx5_esw_flow_attr slow_attr; 1040f493f155SEli Britstein int out_index; 1041d85cdccbSOr Gerlitz 10425dbe906fSPaul Blakey if (flow->flags & MLX5E_TC_FLOW_OFFLOADED) { 10435dbe906fSPaul Blakey if (flow->flags & MLX5E_TC_FLOW_SLOW) 10445dbe906fSPaul Blakey mlx5e_tc_unoffload_from_slow_path(esw, flow, &slow_attr); 10455dbe906fSPaul Blakey else 10465dbe906fSPaul Blakey mlx5e_tc_unoffload_fdb_rules(esw, flow, attr); 10475dbe906fSPaul Blakey } 1048d85cdccbSOr Gerlitz 1049513f8f7fSOr Gerlitz mlx5_eswitch_del_vlan_action(esw, attr); 1050d85cdccbSOr Gerlitz 1051f493f155SEli Britstein for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) 1052f493f155SEli Britstein if (attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP) { 1053d85cdccbSOr Gerlitz mlx5e_detach_encap(priv, flow); 1054f493f155SEli Britstein break; 1055232c0013SHadar Hen Zion } 1056f493f155SEli Britstein kvfree(attr->parse_attr); 1057d7e75a32SOr Gerlitz 1058513f8f7fSOr Gerlitz if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 10591a9527bbSOr Gerlitz mlx5e_detach_mod_hdr(priv, flow); 1060b8aee822SMark Bloch 1061b8aee822SMark Bloch if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 1062b8aee822SMark Bloch mlx5_fc_destroy(esw->dev, attr->counter); 1063d85cdccbSOr Gerlitz } 1064d85cdccbSOr Gerlitz 1065232c0013SHadar Hen Zion void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv, 1066232c0013SHadar Hen Zion struct mlx5e_encap_entry *e) 1067232c0013SHadar Hen Zion { 10683c37745eSOr Gerlitz struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 10695dbe906fSPaul Blakey struct mlx5_esw_flow_attr slow_attr, *esw_attr; 10706d2a3ed0SOr Gerlitz struct mlx5_flow_handle *rule; 10716d2a3ed0SOr Gerlitz struct mlx5_flow_spec *spec; 107279baaec7SEli Britstein struct encap_flow_item *efi; 1073232c0013SHadar Hen Zion struct mlx5e_tc_flow *flow; 1074232c0013SHadar Hen Zion int err; 1075232c0013SHadar Hen Zion 107654c177caSOz Shlomo err = mlx5_packet_reformat_alloc(priv->mdev, 107754c177caSOz Shlomo e->reformat_type, 1078232c0013SHadar Hen Zion e->encap_size, e->encap_header, 107931ca3648SMark Bloch MLX5_FLOW_NAMESPACE_FDB, 1080232c0013SHadar Hen Zion &e->encap_id); 1081232c0013SHadar Hen Zion if (err) { 1082232c0013SHadar Hen Zion mlx5_core_warn(priv->mdev, "Failed to offload cached encapsulation header, %d\n", 1083232c0013SHadar Hen Zion err); 1084232c0013SHadar Hen Zion return; 1085232c0013SHadar Hen Zion } 1086232c0013SHadar Hen Zion e->flags |= MLX5_ENCAP_ENTRY_VALID; 1087f6dfb4c3SHadar Hen Zion mlx5e_rep_queue_neigh_stats_work(priv); 1088232c0013SHadar Hen Zion 108979baaec7SEli Britstein list_for_each_entry(efi, &e->flows, list) { 109079baaec7SEli Britstein flow = container_of(efi, struct mlx5e_tc_flow, encaps[efi->index]); 10913c37745eSOr Gerlitz esw_attr = flow->esw_attr; 10923c37745eSOr Gerlitz esw_attr->encap_id = e->encap_id; 10936d2a3ed0SOr Gerlitz spec = &esw_attr->parse_attr->spec; 10946d2a3ed0SOr Gerlitz 10955dbe906fSPaul Blakey /* update from slow path rule to encap rule */ 10966d2a3ed0SOr Gerlitz rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, esw_attr); 10976d2a3ed0SOr Gerlitz if (IS_ERR(rule)) { 10986d2a3ed0SOr Gerlitz err = PTR_ERR(rule); 1099232c0013SHadar Hen Zion mlx5_core_warn(priv->mdev, "Failed to update cached encapsulation flow, %d\n", 1100232c0013SHadar Hen Zion err); 1101232c0013SHadar Hen Zion continue; 1102232c0013SHadar Hen Zion } 11035dbe906fSPaul Blakey 11045dbe906fSPaul Blakey mlx5e_tc_unoffload_from_slow_path(esw, flow, &slow_attr); 11055dbe906fSPaul Blakey flow->flags |= MLX5E_TC_FLOW_OFFLOADED; /* was unset when slow path rule removed */ 11066d2a3ed0SOr Gerlitz flow->rule[0] = rule; 1107232c0013SHadar Hen Zion } 1108232c0013SHadar Hen Zion } 1109232c0013SHadar Hen Zion 1110232c0013SHadar Hen Zion void mlx5e_tc_encap_flows_del(struct mlx5e_priv *priv, 1111232c0013SHadar Hen Zion struct mlx5e_encap_entry *e) 1112232c0013SHadar Hen Zion { 11133c37745eSOr Gerlitz struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 11145dbe906fSPaul Blakey struct mlx5_esw_flow_attr slow_attr; 11155dbe906fSPaul Blakey struct mlx5_flow_handle *rule; 11165dbe906fSPaul Blakey struct mlx5_flow_spec *spec; 111779baaec7SEli Britstein struct encap_flow_item *efi; 1118232c0013SHadar Hen Zion struct mlx5e_tc_flow *flow; 11195dbe906fSPaul Blakey int err; 1120232c0013SHadar Hen Zion 112179baaec7SEli Britstein list_for_each_entry(efi, &e->flows, list) { 112279baaec7SEli Britstein flow = container_of(efi, struct mlx5e_tc_flow, encaps[efi->index]); 11235dbe906fSPaul Blakey spec = &flow->esw_attr->parse_attr->spec; 11245dbe906fSPaul Blakey 11255dbe906fSPaul Blakey /* update from encap rule to slow path rule */ 11265dbe906fSPaul Blakey rule = mlx5e_tc_offload_to_slow_path(esw, flow, spec, &slow_attr); 11275dbe906fSPaul Blakey 11285dbe906fSPaul Blakey if (IS_ERR(rule)) { 11295dbe906fSPaul Blakey err = PTR_ERR(rule); 11305dbe906fSPaul Blakey mlx5_core_warn(priv->mdev, "Failed to update slow path (encap) flow, %d\n", 11315dbe906fSPaul Blakey err); 11325dbe906fSPaul Blakey continue; 11335dbe906fSPaul Blakey } 11345dbe906fSPaul Blakey 11356d2a3ed0SOr Gerlitz mlx5e_tc_unoffload_fdb_rules(esw, flow, flow->esw_attr); 11365dbe906fSPaul Blakey flow->flags |= MLX5E_TC_FLOW_OFFLOADED; /* was unset when fast path rule removed */ 11375dbe906fSPaul Blakey flow->rule[0] = rule; 1138232c0013SHadar Hen Zion } 1139232c0013SHadar Hen Zion 1140232c0013SHadar Hen Zion if (e->flags & MLX5_ENCAP_ENTRY_VALID) { 1141232c0013SHadar Hen Zion e->flags &= ~MLX5_ENCAP_ENTRY_VALID; 114260786f09SMark Bloch mlx5_packet_reformat_dealloc(priv->mdev, e->encap_id); 1143232c0013SHadar Hen Zion } 1144232c0013SHadar Hen Zion } 1145232c0013SHadar Hen Zion 1146b8aee822SMark Bloch static struct mlx5_fc *mlx5e_tc_get_counter(struct mlx5e_tc_flow *flow) 1147b8aee822SMark Bloch { 1148b8aee822SMark Bloch if (flow->flags & MLX5E_TC_FLOW_ESWITCH) 1149b8aee822SMark Bloch return flow->esw_attr->counter; 1150b8aee822SMark Bloch else 1151b8aee822SMark Bloch return flow->nic_attr->counter; 1152b8aee822SMark Bloch } 1153b8aee822SMark Bloch 1154f6dfb4c3SHadar Hen Zion void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe) 1155f6dfb4c3SHadar Hen Zion { 1156f6dfb4c3SHadar Hen Zion struct mlx5e_neigh *m_neigh = &nhe->m_neigh; 1157f6dfb4c3SHadar Hen Zion u64 bytes, packets, lastuse = 0; 1158f6dfb4c3SHadar Hen Zion struct mlx5e_tc_flow *flow; 1159f6dfb4c3SHadar Hen Zion struct mlx5e_encap_entry *e; 1160f6dfb4c3SHadar Hen Zion struct mlx5_fc *counter; 1161f6dfb4c3SHadar Hen Zion struct neigh_table *tbl; 1162f6dfb4c3SHadar Hen Zion bool neigh_used = false; 1163f6dfb4c3SHadar Hen Zion struct neighbour *n; 1164f6dfb4c3SHadar Hen Zion 1165f6dfb4c3SHadar Hen Zion if (m_neigh->family == AF_INET) 1166f6dfb4c3SHadar Hen Zion tbl = &arp_tbl; 1167f6dfb4c3SHadar Hen Zion #if IS_ENABLED(CONFIG_IPV6) 1168f6dfb4c3SHadar Hen Zion else if (m_neigh->family == AF_INET6) 1169423c9db2SOr Gerlitz tbl = &nd_tbl; 1170f6dfb4c3SHadar Hen Zion #endif 1171f6dfb4c3SHadar Hen Zion else 1172f6dfb4c3SHadar Hen Zion return; 1173f6dfb4c3SHadar Hen Zion 1174f6dfb4c3SHadar Hen Zion list_for_each_entry(e, &nhe->encap_list, encap_list) { 117579baaec7SEli Britstein struct encap_flow_item *efi; 1176f6dfb4c3SHadar Hen Zion if (!(e->flags & MLX5_ENCAP_ENTRY_VALID)) 1177f6dfb4c3SHadar Hen Zion continue; 117879baaec7SEli Britstein list_for_each_entry(efi, &e->flows, list) { 117979baaec7SEli Britstein flow = container_of(efi, struct mlx5e_tc_flow, 118079baaec7SEli Britstein encaps[efi->index]); 1181f6dfb4c3SHadar Hen Zion if (flow->flags & MLX5E_TC_FLOW_OFFLOADED) { 1182b8aee822SMark Bloch counter = mlx5e_tc_get_counter(flow); 1183f6dfb4c3SHadar Hen Zion mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse); 1184f6dfb4c3SHadar Hen Zion if (time_after((unsigned long)lastuse, nhe->reported_lastuse)) { 1185f6dfb4c3SHadar Hen Zion neigh_used = true; 1186f6dfb4c3SHadar Hen Zion break; 1187f6dfb4c3SHadar Hen Zion } 1188f6dfb4c3SHadar Hen Zion } 1189f6dfb4c3SHadar Hen Zion } 1190e36d4810SRoi Dayan if (neigh_used) 1191e36d4810SRoi Dayan break; 1192f6dfb4c3SHadar Hen Zion } 1193f6dfb4c3SHadar Hen Zion 1194f6dfb4c3SHadar Hen Zion if (neigh_used) { 1195f6dfb4c3SHadar Hen Zion nhe->reported_lastuse = jiffies; 1196f6dfb4c3SHadar Hen Zion 1197f6dfb4c3SHadar Hen Zion /* find the relevant neigh according to the cached device and 1198f6dfb4c3SHadar Hen Zion * dst ip pair 1199f6dfb4c3SHadar Hen Zion */ 1200f6dfb4c3SHadar Hen Zion n = neigh_lookup(tbl, &m_neigh->dst_ip, m_neigh->dev); 1201c7f7ba8dSRoi Dayan if (!n) 1202f6dfb4c3SHadar Hen Zion return; 1203f6dfb4c3SHadar Hen Zion 1204f6dfb4c3SHadar Hen Zion neigh_event_send(n, NULL); 1205f6dfb4c3SHadar Hen Zion neigh_release(n); 1206f6dfb4c3SHadar Hen Zion } 1207f6dfb4c3SHadar Hen Zion } 1208f6dfb4c3SHadar Hen Zion 1209d85cdccbSOr Gerlitz static void mlx5e_detach_encap(struct mlx5e_priv *priv, 1210d85cdccbSOr Gerlitz struct mlx5e_tc_flow *flow) 1211d85cdccbSOr Gerlitz { 121279baaec7SEli Britstein struct list_head *next = flow->encaps[0].list.next; 12135067b602SRoi Dayan 121479baaec7SEli Britstein list_del(&flow->encaps[0].list); 12155067b602SRoi Dayan if (list_empty(next)) { 1216c1ae1152SOr Gerlitz struct mlx5e_encap_entry *e; 12175067b602SRoi Dayan 1218c1ae1152SOr Gerlitz e = list_entry(next, struct mlx5e_encap_entry, flows); 1219232c0013SHadar Hen Zion mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e); 1220232c0013SHadar Hen Zion 1221232c0013SHadar Hen Zion if (e->flags & MLX5_ENCAP_ENTRY_VALID) 122260786f09SMark Bloch mlx5_packet_reformat_dealloc(priv->mdev, e->encap_id); 1223232c0013SHadar Hen Zion 1224cdc5a7f3SOr Gerlitz hash_del_rcu(&e->encap_hlist); 1225232c0013SHadar Hen Zion kfree(e->encap_header); 12265067b602SRoi Dayan kfree(e); 12275067b602SRoi Dayan } 12285067b602SRoi Dayan } 12295067b602SRoi Dayan 1230e8f887acSAmir Vadai static void mlx5e_tc_del_flow(struct mlx5e_priv *priv, 1231961e8979SRoi Dayan struct mlx5e_tc_flow *flow) 1232e8f887acSAmir Vadai { 1233d85cdccbSOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_ESWITCH) 1234d85cdccbSOr Gerlitz mlx5e_tc_del_fdb_flow(priv, flow); 1235d85cdccbSOr Gerlitz else 1236d85cdccbSOr Gerlitz mlx5e_tc_del_nic_flow(priv, flow); 1237e8f887acSAmir Vadai } 1238e8f887acSAmir Vadai 1239bbd00f7eSHadar Hen Zion 1240bbd00f7eSHadar Hen Zion static int parse_tunnel_attr(struct mlx5e_priv *priv, 1241bbd00f7eSHadar Hen Zion struct mlx5_flow_spec *spec, 124254c177caSOz Shlomo struct tc_cls_flower_offload *f, 124354c177caSOz Shlomo struct net_device *filter_dev) 1244bbd00f7eSHadar Hen Zion { 1245e98bedf5SEli Britstein struct netlink_ext_ack *extack = f->common.extack; 1246bbd00f7eSHadar Hen Zion void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1247bbd00f7eSHadar Hen Zion outer_headers); 1248bbd00f7eSHadar Hen Zion void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1249bbd00f7eSHadar Hen Zion outer_headers); 1250bbd00f7eSHadar Hen Zion 12512e72eb43SOr Gerlitz struct flow_dissector_key_control *enc_control = 12522e72eb43SOr Gerlitz skb_flow_dissector_target(f->dissector, 12532e72eb43SOr Gerlitz FLOW_DISSECTOR_KEY_ENC_CONTROL, 12542e72eb43SOr Gerlitz f->key); 12554d70564dSOz Shlomo int err = 0; 1256bbd00f7eSHadar Hen Zion 1257101f4de9SOz Shlomo err = mlx5e_tc_tun_parse(filter_dev, priv, spec, f, 12584d70564dSOz Shlomo headers_c, headers_v); 125954c177caSOz Shlomo if (err) { 126054c177caSOz Shlomo NL_SET_ERR_MSG_MOD(extack, 126154c177caSOz Shlomo "failed to parse tunnel attributes"); 1262101f4de9SOz Shlomo return err; 1263bbd00f7eSHadar Hen Zion } 1264bbd00f7eSHadar Hen Zion 12652e72eb43SOr Gerlitz if (enc_control->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1266bbd00f7eSHadar Hen Zion struct flow_dissector_key_ipv4_addrs *key = 1267bbd00f7eSHadar Hen Zion skb_flow_dissector_target(f->dissector, 1268bbd00f7eSHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, 1269bbd00f7eSHadar Hen Zion f->key); 1270bbd00f7eSHadar Hen Zion struct flow_dissector_key_ipv4_addrs *mask = 1271bbd00f7eSHadar Hen Zion skb_flow_dissector_target(f->dissector, 1272bbd00f7eSHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, 1273bbd00f7eSHadar Hen Zion f->mask); 1274bbd00f7eSHadar Hen Zion MLX5_SET(fte_match_set_lyr_2_4, headers_c, 1275bbd00f7eSHadar Hen Zion src_ipv4_src_ipv6.ipv4_layout.ipv4, 1276bbd00f7eSHadar Hen Zion ntohl(mask->src)); 1277bbd00f7eSHadar Hen Zion MLX5_SET(fte_match_set_lyr_2_4, headers_v, 1278bbd00f7eSHadar Hen Zion src_ipv4_src_ipv6.ipv4_layout.ipv4, 1279bbd00f7eSHadar Hen Zion ntohl(key->src)); 1280bbd00f7eSHadar Hen Zion 1281bbd00f7eSHadar Hen Zion MLX5_SET(fte_match_set_lyr_2_4, headers_c, 1282bbd00f7eSHadar Hen Zion dst_ipv4_dst_ipv6.ipv4_layout.ipv4, 1283bbd00f7eSHadar Hen Zion ntohl(mask->dst)); 1284bbd00f7eSHadar Hen Zion MLX5_SET(fte_match_set_lyr_2_4, headers_v, 1285bbd00f7eSHadar Hen Zion dst_ipv4_dst_ipv6.ipv4_layout.ipv4, 1286bbd00f7eSHadar Hen Zion ntohl(key->dst)); 1287bbd00f7eSHadar Hen Zion 1288bbd00f7eSHadar Hen Zion MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype); 1289bbd00f7eSHadar Hen Zion MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IP); 129019f44401SOr Gerlitz } else if (enc_control->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 129119f44401SOr Gerlitz struct flow_dissector_key_ipv6_addrs *key = 129219f44401SOr Gerlitz skb_flow_dissector_target(f->dissector, 129319f44401SOr Gerlitz FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, 129419f44401SOr Gerlitz f->key); 129519f44401SOr Gerlitz struct flow_dissector_key_ipv6_addrs *mask = 129619f44401SOr Gerlitz skb_flow_dissector_target(f->dissector, 129719f44401SOr Gerlitz FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, 129819f44401SOr Gerlitz f->mask); 129919f44401SOr Gerlitz 130019f44401SOr Gerlitz memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 130119f44401SOr Gerlitz src_ipv4_src_ipv6.ipv6_layout.ipv6), 130219f44401SOr Gerlitz &mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6)); 130319f44401SOr Gerlitz memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 130419f44401SOr Gerlitz src_ipv4_src_ipv6.ipv6_layout.ipv6), 130519f44401SOr Gerlitz &key->src, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6)); 130619f44401SOr Gerlitz 130719f44401SOr Gerlitz memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 130819f44401SOr Gerlitz dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 130919f44401SOr Gerlitz &mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6)); 131019f44401SOr Gerlitz memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 131119f44401SOr Gerlitz dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 131219f44401SOr Gerlitz &key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6)); 131319f44401SOr Gerlitz 131419f44401SOr Gerlitz MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype); 131519f44401SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IPV6); 13162e72eb43SOr Gerlitz } 1317bbd00f7eSHadar Hen Zion 1318bcef735cSOr Gerlitz if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_IP)) { 1319bcef735cSOr Gerlitz struct flow_dissector_key_ip *key = 1320bcef735cSOr Gerlitz skb_flow_dissector_target(f->dissector, 1321bcef735cSOr Gerlitz FLOW_DISSECTOR_KEY_ENC_IP, 1322bcef735cSOr Gerlitz f->key); 1323bcef735cSOr Gerlitz struct flow_dissector_key_ip *mask = 1324bcef735cSOr Gerlitz skb_flow_dissector_target(f->dissector, 1325bcef735cSOr Gerlitz FLOW_DISSECTOR_KEY_ENC_IP, 1326bcef735cSOr Gerlitz f->mask); 1327bcef735cSOr Gerlitz 1328bcef735cSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn, mask->tos & 0x3); 1329bcef735cSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, key->tos & 0x3); 1330bcef735cSOr Gerlitz 1331bcef735cSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp, mask->tos >> 2); 1332bcef735cSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, key->tos >> 2); 1333bcef735cSOr Gerlitz 1334bcef735cSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit, mask->ttl); 1335bcef735cSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit, key->ttl); 1336e98bedf5SEli Britstein 1337e98bedf5SEli Britstein if (mask->ttl && 1338e98bedf5SEli Britstein !MLX5_CAP_ESW_FLOWTABLE_FDB 1339e98bedf5SEli Britstein (priv->mdev, 1340e98bedf5SEli Britstein ft_field_support.outer_ipv4_ttl)) { 1341e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 1342e98bedf5SEli Britstein "Matching on TTL is not supported"); 1343e98bedf5SEli Britstein return -EOPNOTSUPP; 1344e98bedf5SEli Britstein } 1345e98bedf5SEli Britstein 1346bcef735cSOr Gerlitz } 1347bcef735cSOr Gerlitz 1348bbd00f7eSHadar Hen Zion /* Enforce DMAC when offloading incoming tunneled flows. 1349bbd00f7eSHadar Hen Zion * Flow counters require a match on the DMAC. 1350bbd00f7eSHadar Hen Zion */ 1351bbd00f7eSHadar Hen Zion MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_47_16); 1352bbd00f7eSHadar Hen Zion MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_15_0); 1353bbd00f7eSHadar Hen Zion ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1354bbd00f7eSHadar Hen Zion dmac_47_16), priv->netdev->dev_addr); 1355bbd00f7eSHadar Hen Zion 1356bbd00f7eSHadar Hen Zion /* let software handle IP fragments */ 1357bbd00f7eSHadar Hen Zion MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1); 1358bbd00f7eSHadar Hen Zion MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0); 1359bbd00f7eSHadar Hen Zion 1360bbd00f7eSHadar Hen Zion return 0; 1361bbd00f7eSHadar Hen Zion } 1362bbd00f7eSHadar Hen Zion 1363de0af0bfSRoi Dayan static int __parse_cls_flower(struct mlx5e_priv *priv, 1364de0af0bfSRoi Dayan struct mlx5_flow_spec *spec, 1365de0af0bfSRoi Dayan struct tc_cls_flower_offload *f, 136654c177caSOz Shlomo struct net_device *filter_dev, 1367d708f902SOr Gerlitz u8 *match_level) 1368e3a2b7edSAmir Vadai { 1369e98bedf5SEli Britstein struct netlink_ext_ack *extack = f->common.extack; 1370c5bb1730SMaor Gottlieb void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1371c5bb1730SMaor Gottlieb outer_headers); 1372c5bb1730SMaor Gottlieb void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1373c5bb1730SMaor Gottlieb outer_headers); 1374699e96ddSJianbo Liu void *misc_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1375699e96ddSJianbo Liu misc_parameters); 1376699e96ddSJianbo Liu void *misc_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1377699e96ddSJianbo Liu misc_parameters); 1378e3a2b7edSAmir Vadai u16 addr_type = 0; 1379e3a2b7edSAmir Vadai u8 ip_proto = 0; 1380e3a2b7edSAmir Vadai 1381d708f902SOr Gerlitz *match_level = MLX5_MATCH_NONE; 1382de0af0bfSRoi Dayan 1383e3a2b7edSAmir Vadai if (f->dissector->used_keys & 1384e3a2b7edSAmir Vadai ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 1385e3a2b7edSAmir Vadai BIT(FLOW_DISSECTOR_KEY_BASIC) | 1386e3a2b7edSAmir Vadai BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 1387095b6cfdSOr Gerlitz BIT(FLOW_DISSECTOR_KEY_VLAN) | 1388699e96ddSJianbo Liu BIT(FLOW_DISSECTOR_KEY_CVLAN) | 1389e3a2b7edSAmir Vadai BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 1390e3a2b7edSAmir Vadai BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 1391bbd00f7eSHadar Hen Zion BIT(FLOW_DISSECTOR_KEY_PORTS) | 1392bbd00f7eSHadar Hen Zion BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | 1393bbd00f7eSHadar Hen Zion BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | 1394bbd00f7eSHadar Hen Zion BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | 1395bbd00f7eSHadar Hen Zion BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | 1396e77834ecSOr Gerlitz BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | 1397fd7da28bSOr Gerlitz BIT(FLOW_DISSECTOR_KEY_TCP) | 1398bcef735cSOr Gerlitz BIT(FLOW_DISSECTOR_KEY_IP) | 1399bcef735cSOr Gerlitz BIT(FLOW_DISSECTOR_KEY_ENC_IP))) { 1400e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, "Unsupported key"); 1401e3a2b7edSAmir Vadai netdev_warn(priv->netdev, "Unsupported key used: 0x%x\n", 1402e3a2b7edSAmir Vadai f->dissector->used_keys); 1403e3a2b7edSAmir Vadai return -EOPNOTSUPP; 1404e3a2b7edSAmir Vadai } 1405e3a2b7edSAmir Vadai 1406bbd00f7eSHadar Hen Zion if ((dissector_uses_key(f->dissector, 1407bbd00f7eSHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) || 1408bbd00f7eSHadar Hen Zion dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID) || 1409bbd00f7eSHadar Hen Zion dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) && 1410bbd00f7eSHadar Hen Zion dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) { 1411bbd00f7eSHadar Hen Zion struct flow_dissector_key_control *key = 1412bbd00f7eSHadar Hen Zion skb_flow_dissector_target(f->dissector, 1413bbd00f7eSHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_CONTROL, 1414bbd00f7eSHadar Hen Zion f->key); 1415bbd00f7eSHadar Hen Zion switch (key->addr_type) { 1416bbd00f7eSHadar Hen Zion case FLOW_DISSECTOR_KEY_IPV4_ADDRS: 141719f44401SOr Gerlitz case FLOW_DISSECTOR_KEY_IPV6_ADDRS: 141854c177caSOz Shlomo if (parse_tunnel_attr(priv, spec, f, filter_dev)) 1419bbd00f7eSHadar Hen Zion return -EOPNOTSUPP; 1420bbd00f7eSHadar Hen Zion break; 1421bbd00f7eSHadar Hen Zion default: 1422bbd00f7eSHadar Hen Zion return -EOPNOTSUPP; 1423bbd00f7eSHadar Hen Zion } 1424bbd00f7eSHadar Hen Zion 1425bbd00f7eSHadar Hen Zion /* In decap flow, header pointers should point to the inner 1426bbd00f7eSHadar Hen Zion * headers, outer header were already set by parse_tunnel_attr 1427bbd00f7eSHadar Hen Zion */ 1428bbd00f7eSHadar Hen Zion headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1429bbd00f7eSHadar Hen Zion inner_headers); 1430bbd00f7eSHadar Hen Zion headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1431bbd00f7eSHadar Hen Zion inner_headers); 1432bbd00f7eSHadar Hen Zion } 1433bbd00f7eSHadar Hen Zion 1434d3a80bb5SOr Gerlitz if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) { 1435d3a80bb5SOr Gerlitz struct flow_dissector_key_basic *key = 1436e3a2b7edSAmir Vadai skb_flow_dissector_target(f->dissector, 1437d3a80bb5SOr Gerlitz FLOW_DISSECTOR_KEY_BASIC, 1438e3a2b7edSAmir Vadai f->key); 1439d3a80bb5SOr Gerlitz struct flow_dissector_key_basic *mask = 1440e3a2b7edSAmir Vadai skb_flow_dissector_target(f->dissector, 1441d3a80bb5SOr Gerlitz FLOW_DISSECTOR_KEY_BASIC, 1442e3a2b7edSAmir Vadai f->mask); 1443d3a80bb5SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype, 1444d3a80bb5SOr Gerlitz ntohs(mask->n_proto)); 1445d3a80bb5SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 1446d3a80bb5SOr Gerlitz ntohs(key->n_proto)); 1447e3a2b7edSAmir Vadai 1448d3a80bb5SOr Gerlitz if (mask->n_proto) 1449d708f902SOr Gerlitz *match_level = MLX5_MATCH_L2; 1450e3a2b7edSAmir Vadai } 1451e3a2b7edSAmir Vadai 1452095b6cfdSOr Gerlitz if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) { 1453095b6cfdSOr Gerlitz struct flow_dissector_key_vlan *key = 1454095b6cfdSOr Gerlitz skb_flow_dissector_target(f->dissector, 1455095b6cfdSOr Gerlitz FLOW_DISSECTOR_KEY_VLAN, 1456095b6cfdSOr Gerlitz f->key); 1457095b6cfdSOr Gerlitz struct flow_dissector_key_vlan *mask = 1458095b6cfdSOr Gerlitz skb_flow_dissector_target(f->dissector, 1459095b6cfdSOr Gerlitz FLOW_DISSECTOR_KEY_VLAN, 1460095b6cfdSOr Gerlitz f->mask); 1461699e96ddSJianbo Liu if (mask->vlan_id || mask->vlan_priority || mask->vlan_tpid) { 1462699e96ddSJianbo Liu if (key->vlan_tpid == htons(ETH_P_8021AD)) { 1463699e96ddSJianbo Liu MLX5_SET(fte_match_set_lyr_2_4, headers_c, 1464699e96ddSJianbo Liu svlan_tag, 1); 1465699e96ddSJianbo Liu MLX5_SET(fte_match_set_lyr_2_4, headers_v, 1466699e96ddSJianbo Liu svlan_tag, 1); 1467699e96ddSJianbo Liu } else { 1468699e96ddSJianbo Liu MLX5_SET(fte_match_set_lyr_2_4, headers_c, 1469699e96ddSJianbo Liu cvlan_tag, 1); 1470699e96ddSJianbo Liu MLX5_SET(fte_match_set_lyr_2_4, headers_v, 1471699e96ddSJianbo Liu cvlan_tag, 1); 1472699e96ddSJianbo Liu } 1473095b6cfdSOr Gerlitz 1474095b6cfdSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid, mask->vlan_id); 1475095b6cfdSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, key->vlan_id); 1476358d79a4SOr Gerlitz 1477358d79a4SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_prio, mask->vlan_priority); 1478358d79a4SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, key->vlan_priority); 147954782900SOr Gerlitz 1480d708f902SOr Gerlitz *match_level = MLX5_MATCH_L2; 1481095b6cfdSOr Gerlitz } 1482d3a80bb5SOr Gerlitz } else if (*match_level != MLX5_MATCH_NONE) { 1483cee26487SJianbo Liu MLX5_SET(fte_match_set_lyr_2_4, headers_c, svlan_tag, 1); 1484cee26487SJianbo Liu MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1); 1485d3a80bb5SOr Gerlitz *match_level = MLX5_MATCH_L2; 1486095b6cfdSOr Gerlitz } 1487095b6cfdSOr Gerlitz 1488699e96ddSJianbo Liu if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CVLAN)) { 1489699e96ddSJianbo Liu struct flow_dissector_key_vlan *key = 1490699e96ddSJianbo Liu skb_flow_dissector_target(f->dissector, 1491699e96ddSJianbo Liu FLOW_DISSECTOR_KEY_CVLAN, 1492699e96ddSJianbo Liu f->key); 1493699e96ddSJianbo Liu struct flow_dissector_key_vlan *mask = 1494699e96ddSJianbo Liu skb_flow_dissector_target(f->dissector, 1495699e96ddSJianbo Liu FLOW_DISSECTOR_KEY_CVLAN, 1496699e96ddSJianbo Liu f->mask); 1497699e96ddSJianbo Liu if (mask->vlan_id || mask->vlan_priority || mask->vlan_tpid) { 1498699e96ddSJianbo Liu if (key->vlan_tpid == htons(ETH_P_8021AD)) { 1499699e96ddSJianbo Liu MLX5_SET(fte_match_set_misc, misc_c, 1500699e96ddSJianbo Liu outer_second_svlan_tag, 1); 1501699e96ddSJianbo Liu MLX5_SET(fte_match_set_misc, misc_v, 1502699e96ddSJianbo Liu outer_second_svlan_tag, 1); 1503699e96ddSJianbo Liu } else { 1504699e96ddSJianbo Liu MLX5_SET(fte_match_set_misc, misc_c, 1505699e96ddSJianbo Liu outer_second_cvlan_tag, 1); 1506699e96ddSJianbo Liu MLX5_SET(fte_match_set_misc, misc_v, 1507699e96ddSJianbo Liu outer_second_cvlan_tag, 1); 1508699e96ddSJianbo Liu } 1509699e96ddSJianbo Liu 1510699e96ddSJianbo Liu MLX5_SET(fte_match_set_misc, misc_c, outer_second_vid, 1511699e96ddSJianbo Liu mask->vlan_id); 1512699e96ddSJianbo Liu MLX5_SET(fte_match_set_misc, misc_v, outer_second_vid, 1513699e96ddSJianbo Liu key->vlan_id); 1514699e96ddSJianbo Liu MLX5_SET(fte_match_set_misc, misc_c, outer_second_prio, 1515699e96ddSJianbo Liu mask->vlan_priority); 1516699e96ddSJianbo Liu MLX5_SET(fte_match_set_misc, misc_v, outer_second_prio, 1517699e96ddSJianbo Liu key->vlan_priority); 1518699e96ddSJianbo Liu 1519699e96ddSJianbo Liu *match_level = MLX5_MATCH_L2; 1520699e96ddSJianbo Liu } 1521699e96ddSJianbo Liu } 1522699e96ddSJianbo Liu 1523d3a80bb5SOr Gerlitz if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 1524d3a80bb5SOr Gerlitz struct flow_dissector_key_eth_addrs *key = 152554782900SOr Gerlitz skb_flow_dissector_target(f->dissector, 1526d3a80bb5SOr Gerlitz FLOW_DISSECTOR_KEY_ETH_ADDRS, 152754782900SOr Gerlitz f->key); 1528d3a80bb5SOr Gerlitz struct flow_dissector_key_eth_addrs *mask = 152954782900SOr Gerlitz skb_flow_dissector_target(f->dissector, 1530d3a80bb5SOr Gerlitz FLOW_DISSECTOR_KEY_ETH_ADDRS, 153154782900SOr Gerlitz f->mask); 153254782900SOr Gerlitz 1533d3a80bb5SOr Gerlitz ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 1534d3a80bb5SOr Gerlitz dmac_47_16), 1535d3a80bb5SOr Gerlitz mask->dst); 1536d3a80bb5SOr Gerlitz ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1537d3a80bb5SOr Gerlitz dmac_47_16), 1538d3a80bb5SOr Gerlitz key->dst); 1539d3a80bb5SOr Gerlitz 1540d3a80bb5SOr Gerlitz ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 1541d3a80bb5SOr Gerlitz smac_47_16), 1542d3a80bb5SOr Gerlitz mask->src); 1543d3a80bb5SOr Gerlitz ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1544d3a80bb5SOr Gerlitz smac_47_16), 1545d3a80bb5SOr Gerlitz key->src); 1546d3a80bb5SOr Gerlitz 1547d3a80bb5SOr Gerlitz if (!is_zero_ether_addr(mask->src) || !is_zero_ether_addr(mask->dst)) 1548d708f902SOr Gerlitz *match_level = MLX5_MATCH_L2; 154954782900SOr Gerlitz } 155054782900SOr Gerlitz 155154782900SOr Gerlitz if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) { 155254782900SOr Gerlitz struct flow_dissector_key_control *key = 155354782900SOr Gerlitz skb_flow_dissector_target(f->dissector, 155454782900SOr Gerlitz FLOW_DISSECTOR_KEY_CONTROL, 155554782900SOr Gerlitz f->key); 155654782900SOr Gerlitz 155754782900SOr Gerlitz struct flow_dissector_key_control *mask = 155854782900SOr Gerlitz skb_flow_dissector_target(f->dissector, 155954782900SOr Gerlitz FLOW_DISSECTOR_KEY_CONTROL, 156054782900SOr Gerlitz f->mask); 156154782900SOr Gerlitz addr_type = key->addr_type; 156254782900SOr Gerlitz 156354782900SOr Gerlitz /* the HW doesn't support frag first/later */ 156454782900SOr Gerlitz if (mask->flags & FLOW_DIS_FIRST_FRAG) 156554782900SOr Gerlitz return -EOPNOTSUPP; 156654782900SOr Gerlitz 156754782900SOr Gerlitz if (mask->flags & FLOW_DIS_IS_FRAGMENT) { 156854782900SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1); 156954782900SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 157054782900SOr Gerlitz key->flags & FLOW_DIS_IS_FRAGMENT); 157154782900SOr Gerlitz 157254782900SOr Gerlitz /* the HW doesn't need L3 inline to match on frag=no */ 157354782900SOr Gerlitz if (!(key->flags & FLOW_DIS_IS_FRAGMENT)) 157483621b7dSOr Gerlitz *match_level = MLX5_MATCH_L2; 157554782900SOr Gerlitz /* *** L2 attributes parsing up to here *** */ 157654782900SOr Gerlitz else 157783621b7dSOr Gerlitz *match_level = MLX5_MATCH_L3; 157854782900SOr Gerlitz } 157954782900SOr Gerlitz } 158054782900SOr Gerlitz 158154782900SOr Gerlitz if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) { 158254782900SOr Gerlitz struct flow_dissector_key_basic *key = 158354782900SOr Gerlitz skb_flow_dissector_target(f->dissector, 158454782900SOr Gerlitz FLOW_DISSECTOR_KEY_BASIC, 158554782900SOr Gerlitz f->key); 158654782900SOr Gerlitz struct flow_dissector_key_basic *mask = 158754782900SOr Gerlitz skb_flow_dissector_target(f->dissector, 158854782900SOr Gerlitz FLOW_DISSECTOR_KEY_BASIC, 158954782900SOr Gerlitz f->mask); 159054782900SOr Gerlitz ip_proto = key->ip_proto; 159154782900SOr Gerlitz 159254782900SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 159354782900SOr Gerlitz mask->ip_proto); 159454782900SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 159554782900SOr Gerlitz key->ip_proto); 159654782900SOr Gerlitz 159754782900SOr Gerlitz if (mask->ip_proto) 1598d708f902SOr Gerlitz *match_level = MLX5_MATCH_L3; 159954782900SOr Gerlitz } 160054782900SOr Gerlitz 1601e3a2b7edSAmir Vadai if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1602e3a2b7edSAmir Vadai struct flow_dissector_key_ipv4_addrs *key = 1603e3a2b7edSAmir Vadai skb_flow_dissector_target(f->dissector, 1604e3a2b7edSAmir Vadai FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1605e3a2b7edSAmir Vadai f->key); 1606e3a2b7edSAmir Vadai struct flow_dissector_key_ipv4_addrs *mask = 1607e3a2b7edSAmir Vadai skb_flow_dissector_target(f->dissector, 1608e3a2b7edSAmir Vadai FLOW_DISSECTOR_KEY_IPV4_ADDRS, 1609e3a2b7edSAmir Vadai f->mask); 1610e3a2b7edSAmir Vadai 1611e3a2b7edSAmir Vadai memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 1612e3a2b7edSAmir Vadai src_ipv4_src_ipv6.ipv4_layout.ipv4), 1613e3a2b7edSAmir Vadai &mask->src, sizeof(mask->src)); 1614e3a2b7edSAmir Vadai memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1615e3a2b7edSAmir Vadai src_ipv4_src_ipv6.ipv4_layout.ipv4), 1616e3a2b7edSAmir Vadai &key->src, sizeof(key->src)); 1617e3a2b7edSAmir Vadai memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 1618e3a2b7edSAmir Vadai dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 1619e3a2b7edSAmir Vadai &mask->dst, sizeof(mask->dst)); 1620e3a2b7edSAmir Vadai memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1621e3a2b7edSAmir Vadai dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 1622e3a2b7edSAmir Vadai &key->dst, sizeof(key->dst)); 1623de0af0bfSRoi Dayan 1624de0af0bfSRoi Dayan if (mask->src || mask->dst) 1625d708f902SOr Gerlitz *match_level = MLX5_MATCH_L3; 1626e3a2b7edSAmir Vadai } 1627e3a2b7edSAmir Vadai 1628e3a2b7edSAmir Vadai if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 1629e3a2b7edSAmir Vadai struct flow_dissector_key_ipv6_addrs *key = 1630e3a2b7edSAmir Vadai skb_flow_dissector_target(f->dissector, 1631e3a2b7edSAmir Vadai FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1632e3a2b7edSAmir Vadai f->key); 1633e3a2b7edSAmir Vadai struct flow_dissector_key_ipv6_addrs *mask = 1634e3a2b7edSAmir Vadai skb_flow_dissector_target(f->dissector, 1635e3a2b7edSAmir Vadai FLOW_DISSECTOR_KEY_IPV6_ADDRS, 1636e3a2b7edSAmir Vadai f->mask); 1637e3a2b7edSAmir Vadai 1638e3a2b7edSAmir Vadai memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 1639e3a2b7edSAmir Vadai src_ipv4_src_ipv6.ipv6_layout.ipv6), 1640e3a2b7edSAmir Vadai &mask->src, sizeof(mask->src)); 1641e3a2b7edSAmir Vadai memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1642e3a2b7edSAmir Vadai src_ipv4_src_ipv6.ipv6_layout.ipv6), 1643e3a2b7edSAmir Vadai &key->src, sizeof(key->src)); 1644e3a2b7edSAmir Vadai 1645e3a2b7edSAmir Vadai memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 1646e3a2b7edSAmir Vadai dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 1647e3a2b7edSAmir Vadai &mask->dst, sizeof(mask->dst)); 1648e3a2b7edSAmir Vadai memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 1649e3a2b7edSAmir Vadai dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 1650e3a2b7edSAmir Vadai &key->dst, sizeof(key->dst)); 1651de0af0bfSRoi Dayan 1652de0af0bfSRoi Dayan if (ipv6_addr_type(&mask->src) != IPV6_ADDR_ANY || 1653de0af0bfSRoi Dayan ipv6_addr_type(&mask->dst) != IPV6_ADDR_ANY) 1654d708f902SOr Gerlitz *match_level = MLX5_MATCH_L3; 1655e3a2b7edSAmir Vadai } 1656e3a2b7edSAmir Vadai 16571f97a526SOr Gerlitz if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_IP)) { 16581f97a526SOr Gerlitz struct flow_dissector_key_ip *key = 16591f97a526SOr Gerlitz skb_flow_dissector_target(f->dissector, 16601f97a526SOr Gerlitz FLOW_DISSECTOR_KEY_IP, 16611f97a526SOr Gerlitz f->key); 16621f97a526SOr Gerlitz struct flow_dissector_key_ip *mask = 16631f97a526SOr Gerlitz skb_flow_dissector_target(f->dissector, 16641f97a526SOr Gerlitz FLOW_DISSECTOR_KEY_IP, 16651f97a526SOr Gerlitz f->mask); 16661f97a526SOr Gerlitz 16671f97a526SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn, mask->tos & 0x3); 16681f97a526SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, key->tos & 0x3); 16691f97a526SOr Gerlitz 16701f97a526SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp, mask->tos >> 2); 16711f97a526SOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, key->tos >> 2); 16721f97a526SOr Gerlitz 1673a8ade55fSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit, mask->ttl); 1674a8ade55fSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit, key->ttl); 16751f97a526SOr Gerlitz 1676a8ade55fSOr Gerlitz if (mask->ttl && 1677a8ade55fSOr Gerlitz !MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev, 1678e98bedf5SEli Britstein ft_field_support.outer_ipv4_ttl)) { 1679e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 1680e98bedf5SEli Britstein "Matching on TTL is not supported"); 16811f97a526SOr Gerlitz return -EOPNOTSUPP; 1682e98bedf5SEli Britstein } 1683a8ade55fSOr Gerlitz 1684a8ade55fSOr Gerlitz if (mask->tos || mask->ttl) 1685d708f902SOr Gerlitz *match_level = MLX5_MATCH_L3; 16861f97a526SOr Gerlitz } 16871f97a526SOr Gerlitz 168854782900SOr Gerlitz /* *** L3 attributes parsing up to here *** */ 168954782900SOr Gerlitz 1690e3a2b7edSAmir Vadai if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) { 1691e3a2b7edSAmir Vadai struct flow_dissector_key_ports *key = 1692e3a2b7edSAmir Vadai skb_flow_dissector_target(f->dissector, 1693e3a2b7edSAmir Vadai FLOW_DISSECTOR_KEY_PORTS, 1694e3a2b7edSAmir Vadai f->key); 1695e3a2b7edSAmir Vadai struct flow_dissector_key_ports *mask = 1696e3a2b7edSAmir Vadai skb_flow_dissector_target(f->dissector, 1697e3a2b7edSAmir Vadai FLOW_DISSECTOR_KEY_PORTS, 1698e3a2b7edSAmir Vadai f->mask); 1699e3a2b7edSAmir Vadai switch (ip_proto) { 1700e3a2b7edSAmir Vadai case IPPROTO_TCP: 1701e3a2b7edSAmir Vadai MLX5_SET(fte_match_set_lyr_2_4, headers_c, 1702e3a2b7edSAmir Vadai tcp_sport, ntohs(mask->src)); 1703e3a2b7edSAmir Vadai MLX5_SET(fte_match_set_lyr_2_4, headers_v, 1704e3a2b7edSAmir Vadai tcp_sport, ntohs(key->src)); 1705e3a2b7edSAmir Vadai 1706e3a2b7edSAmir Vadai MLX5_SET(fte_match_set_lyr_2_4, headers_c, 1707e3a2b7edSAmir Vadai tcp_dport, ntohs(mask->dst)); 1708e3a2b7edSAmir Vadai MLX5_SET(fte_match_set_lyr_2_4, headers_v, 1709e3a2b7edSAmir Vadai tcp_dport, ntohs(key->dst)); 1710e3a2b7edSAmir Vadai break; 1711e3a2b7edSAmir Vadai 1712e3a2b7edSAmir Vadai case IPPROTO_UDP: 1713e3a2b7edSAmir Vadai MLX5_SET(fte_match_set_lyr_2_4, headers_c, 1714e3a2b7edSAmir Vadai udp_sport, ntohs(mask->src)); 1715e3a2b7edSAmir Vadai MLX5_SET(fte_match_set_lyr_2_4, headers_v, 1716e3a2b7edSAmir Vadai udp_sport, ntohs(key->src)); 1717e3a2b7edSAmir Vadai 1718e3a2b7edSAmir Vadai MLX5_SET(fte_match_set_lyr_2_4, headers_c, 1719e3a2b7edSAmir Vadai udp_dport, ntohs(mask->dst)); 1720e3a2b7edSAmir Vadai MLX5_SET(fte_match_set_lyr_2_4, headers_v, 1721e3a2b7edSAmir Vadai udp_dport, ntohs(key->dst)); 1722e3a2b7edSAmir Vadai break; 1723e3a2b7edSAmir Vadai default: 1724e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 1725e98bedf5SEli Britstein "Only UDP and TCP transports are supported for L4 matching"); 1726e3a2b7edSAmir Vadai netdev_err(priv->netdev, 1727e3a2b7edSAmir Vadai "Only UDP and TCP transport are supported\n"); 1728e3a2b7edSAmir Vadai return -EINVAL; 1729e3a2b7edSAmir Vadai } 1730de0af0bfSRoi Dayan 1731de0af0bfSRoi Dayan if (mask->src || mask->dst) 1732d708f902SOr Gerlitz *match_level = MLX5_MATCH_L4; 1733e3a2b7edSAmir Vadai } 1734e3a2b7edSAmir Vadai 1735e77834ecSOr Gerlitz if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_TCP)) { 1736e77834ecSOr Gerlitz struct flow_dissector_key_tcp *key = 1737e77834ecSOr Gerlitz skb_flow_dissector_target(f->dissector, 1738e77834ecSOr Gerlitz FLOW_DISSECTOR_KEY_TCP, 1739e77834ecSOr Gerlitz f->key); 1740e77834ecSOr Gerlitz struct flow_dissector_key_tcp *mask = 1741e77834ecSOr Gerlitz skb_flow_dissector_target(f->dissector, 1742e77834ecSOr Gerlitz FLOW_DISSECTOR_KEY_TCP, 1743e77834ecSOr Gerlitz f->mask); 1744e77834ecSOr Gerlitz 1745e77834ecSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags, 1746e77834ecSOr Gerlitz ntohs(mask->flags)); 1747e77834ecSOr Gerlitz MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags, 1748e77834ecSOr Gerlitz ntohs(key->flags)); 1749e77834ecSOr Gerlitz 1750e77834ecSOr Gerlitz if (mask->flags) 1751d708f902SOr Gerlitz *match_level = MLX5_MATCH_L4; 1752e77834ecSOr Gerlitz } 1753e77834ecSOr Gerlitz 1754e3a2b7edSAmir Vadai return 0; 1755e3a2b7edSAmir Vadai } 1756e3a2b7edSAmir Vadai 1757de0af0bfSRoi Dayan static int parse_cls_flower(struct mlx5e_priv *priv, 175865ba8fb7SOr Gerlitz struct mlx5e_tc_flow *flow, 1759de0af0bfSRoi Dayan struct mlx5_flow_spec *spec, 176054c177caSOz Shlomo struct tc_cls_flower_offload *f, 176154c177caSOz Shlomo struct net_device *filter_dev) 1762de0af0bfSRoi Dayan { 1763e98bedf5SEli Britstein struct netlink_ext_ack *extack = f->common.extack; 1764de0af0bfSRoi Dayan struct mlx5_core_dev *dev = priv->mdev; 1765de0af0bfSRoi Dayan struct mlx5_eswitch *esw = dev->priv.eswitch; 17661d447a39SSaeed Mahameed struct mlx5e_rep_priv *rpriv = priv->ppriv; 17671d447a39SSaeed Mahameed struct mlx5_eswitch_rep *rep; 1768d708f902SOr Gerlitz u8 match_level; 1769de0af0bfSRoi Dayan int err; 1770de0af0bfSRoi Dayan 177154c177caSOz Shlomo err = __parse_cls_flower(priv, spec, f, filter_dev, &match_level); 1772de0af0bfSRoi Dayan 17731d447a39SSaeed Mahameed if (!err && (flow->flags & MLX5E_TC_FLOW_ESWITCH)) { 17741d447a39SSaeed Mahameed rep = rpriv->rep; 17751d447a39SSaeed Mahameed if (rep->vport != FDB_UPLINK_VPORT && 17761d447a39SSaeed Mahameed (esw->offloads.inline_mode != MLX5_INLINE_MODE_NONE && 1777d708f902SOr Gerlitz esw->offloads.inline_mode < match_level)) { 1778e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 1779e98bedf5SEli Britstein "Flow is not offloaded due to min inline setting"); 1780de0af0bfSRoi Dayan netdev_warn(priv->netdev, 1781de0af0bfSRoi Dayan "Flow is not offloaded due to min inline setting, required %d actual %d\n", 1782d708f902SOr Gerlitz match_level, esw->offloads.inline_mode); 1783de0af0bfSRoi Dayan return -EOPNOTSUPP; 1784de0af0bfSRoi Dayan } 1785de0af0bfSRoi Dayan } 1786de0af0bfSRoi Dayan 178738aa51c1SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_ESWITCH) 178838aa51c1SOr Gerlitz flow->esw_attr->match_level = match_level; 178938aa51c1SOr Gerlitz else 179038aa51c1SOr Gerlitz flow->nic_attr->match_level = match_level; 179138aa51c1SOr Gerlitz 1792de0af0bfSRoi Dayan return err; 1793de0af0bfSRoi Dayan } 1794de0af0bfSRoi Dayan 1795d79b6df6SOr Gerlitz struct pedit_headers { 1796d79b6df6SOr Gerlitz struct ethhdr eth; 1797d79b6df6SOr Gerlitz struct iphdr ip4; 1798d79b6df6SOr Gerlitz struct ipv6hdr ip6; 1799d79b6df6SOr Gerlitz struct tcphdr tcp; 1800d79b6df6SOr Gerlitz struct udphdr udp; 1801d79b6df6SOr Gerlitz }; 1802d79b6df6SOr Gerlitz 1803d79b6df6SOr Gerlitz static int pedit_header_offsets[] = { 1804d79b6df6SOr Gerlitz [TCA_PEDIT_KEY_EX_HDR_TYPE_ETH] = offsetof(struct pedit_headers, eth), 1805d79b6df6SOr Gerlitz [TCA_PEDIT_KEY_EX_HDR_TYPE_IP4] = offsetof(struct pedit_headers, ip4), 1806d79b6df6SOr Gerlitz [TCA_PEDIT_KEY_EX_HDR_TYPE_IP6] = offsetof(struct pedit_headers, ip6), 1807d79b6df6SOr Gerlitz [TCA_PEDIT_KEY_EX_HDR_TYPE_TCP] = offsetof(struct pedit_headers, tcp), 1808d79b6df6SOr Gerlitz [TCA_PEDIT_KEY_EX_HDR_TYPE_UDP] = offsetof(struct pedit_headers, udp), 1809d79b6df6SOr Gerlitz }; 1810d79b6df6SOr Gerlitz 1811d79b6df6SOr Gerlitz #define pedit_header(_ph, _htype) ((void *)(_ph) + pedit_header_offsets[_htype]) 1812d79b6df6SOr Gerlitz 1813d79b6df6SOr Gerlitz static int set_pedit_val(u8 hdr_type, u32 mask, u32 val, u32 offset, 1814d79b6df6SOr Gerlitz struct pedit_headers *masks, 1815d79b6df6SOr Gerlitz struct pedit_headers *vals) 1816d79b6df6SOr Gerlitz { 1817d79b6df6SOr Gerlitz u32 *curr_pmask, *curr_pval; 1818d79b6df6SOr Gerlitz 1819d79b6df6SOr Gerlitz if (hdr_type >= __PEDIT_HDR_TYPE_MAX) 1820d79b6df6SOr Gerlitz goto out_err; 1821d79b6df6SOr Gerlitz 1822d79b6df6SOr Gerlitz curr_pmask = (u32 *)(pedit_header(masks, hdr_type) + offset); 1823d79b6df6SOr Gerlitz curr_pval = (u32 *)(pedit_header(vals, hdr_type) + offset); 1824d79b6df6SOr Gerlitz 1825d79b6df6SOr Gerlitz if (*curr_pmask & mask) /* disallow acting twice on the same location */ 1826d79b6df6SOr Gerlitz goto out_err; 1827d79b6df6SOr Gerlitz 1828d79b6df6SOr Gerlitz *curr_pmask |= mask; 1829d79b6df6SOr Gerlitz *curr_pval |= (val & mask); 1830d79b6df6SOr Gerlitz 1831d79b6df6SOr Gerlitz return 0; 1832d79b6df6SOr Gerlitz 1833d79b6df6SOr Gerlitz out_err: 1834d79b6df6SOr Gerlitz return -EOPNOTSUPP; 1835d79b6df6SOr Gerlitz } 1836d79b6df6SOr Gerlitz 1837d79b6df6SOr Gerlitz struct mlx5_fields { 1838d79b6df6SOr Gerlitz u8 field; 1839d79b6df6SOr Gerlitz u8 size; 1840d79b6df6SOr Gerlitz u32 offset; 1841d79b6df6SOr Gerlitz }; 1842d79b6df6SOr Gerlitz 1843a8e4f0c4SOr Gerlitz #define OFFLOAD(fw_field, size, field, off) \ 1844a8e4f0c4SOr Gerlitz {MLX5_ACTION_IN_FIELD_OUT_ ## fw_field, size, offsetof(struct pedit_headers, field) + (off)} 1845a8e4f0c4SOr Gerlitz 1846d79b6df6SOr Gerlitz static struct mlx5_fields fields[] = { 1847a8e4f0c4SOr Gerlitz OFFLOAD(DMAC_47_16, 4, eth.h_dest[0], 0), 1848a8e4f0c4SOr Gerlitz OFFLOAD(DMAC_15_0, 2, eth.h_dest[4], 0), 1849a8e4f0c4SOr Gerlitz OFFLOAD(SMAC_47_16, 4, eth.h_source[0], 0), 1850a8e4f0c4SOr Gerlitz OFFLOAD(SMAC_15_0, 2, eth.h_source[4], 0), 1851a8e4f0c4SOr Gerlitz OFFLOAD(ETHERTYPE, 2, eth.h_proto, 0), 1852d79b6df6SOr Gerlitz 1853a8e4f0c4SOr Gerlitz OFFLOAD(IP_TTL, 1, ip4.ttl, 0), 1854a8e4f0c4SOr Gerlitz OFFLOAD(SIPV4, 4, ip4.saddr, 0), 1855a8e4f0c4SOr Gerlitz OFFLOAD(DIPV4, 4, ip4.daddr, 0), 1856d79b6df6SOr Gerlitz 1857a8e4f0c4SOr Gerlitz OFFLOAD(SIPV6_127_96, 4, ip6.saddr.s6_addr32[0], 0), 1858a8e4f0c4SOr Gerlitz OFFLOAD(SIPV6_95_64, 4, ip6.saddr.s6_addr32[1], 0), 1859a8e4f0c4SOr Gerlitz OFFLOAD(SIPV6_63_32, 4, ip6.saddr.s6_addr32[2], 0), 1860a8e4f0c4SOr Gerlitz OFFLOAD(SIPV6_31_0, 4, ip6.saddr.s6_addr32[3], 0), 1861a8e4f0c4SOr Gerlitz OFFLOAD(DIPV6_127_96, 4, ip6.daddr.s6_addr32[0], 0), 1862a8e4f0c4SOr Gerlitz OFFLOAD(DIPV6_95_64, 4, ip6.daddr.s6_addr32[1], 0), 1863a8e4f0c4SOr Gerlitz OFFLOAD(DIPV6_63_32, 4, ip6.daddr.s6_addr32[2], 0), 1864a8e4f0c4SOr Gerlitz OFFLOAD(DIPV6_31_0, 4, ip6.daddr.s6_addr32[3], 0), 18650c0316f5SOr Gerlitz OFFLOAD(IPV6_HOPLIMIT, 1, ip6.hop_limit, 0), 1866d79b6df6SOr Gerlitz 1867a8e4f0c4SOr Gerlitz OFFLOAD(TCP_SPORT, 2, tcp.source, 0), 1868a8e4f0c4SOr Gerlitz OFFLOAD(TCP_DPORT, 2, tcp.dest, 0), 1869a8e4f0c4SOr Gerlitz OFFLOAD(TCP_FLAGS, 1, tcp.ack_seq, 5), 1870d79b6df6SOr Gerlitz 1871a8e4f0c4SOr Gerlitz OFFLOAD(UDP_SPORT, 2, udp.source, 0), 1872a8e4f0c4SOr Gerlitz OFFLOAD(UDP_DPORT, 2, udp.dest, 0), 1873d79b6df6SOr Gerlitz }; 1874d79b6df6SOr Gerlitz 1875d79b6df6SOr Gerlitz /* On input attr->num_mod_hdr_actions tells how many HW actions can be parsed at 1876d79b6df6SOr Gerlitz * max from the SW pedit action. On success, it says how many HW actions were 1877d79b6df6SOr Gerlitz * actually parsed. 1878d79b6df6SOr Gerlitz */ 1879d79b6df6SOr Gerlitz static int offload_pedit_fields(struct pedit_headers *masks, 1880d79b6df6SOr Gerlitz struct pedit_headers *vals, 1881e98bedf5SEli Britstein struct mlx5e_tc_flow_parse_attr *parse_attr, 1882e98bedf5SEli Britstein struct netlink_ext_ack *extack) 1883d79b6df6SOr Gerlitz { 1884d79b6df6SOr Gerlitz struct pedit_headers *set_masks, *add_masks, *set_vals, *add_vals; 18852b64bebaSOr Gerlitz int i, action_size, nactions, max_actions, first, last, next_z; 1886d79b6df6SOr Gerlitz void *s_masks_p, *a_masks_p, *vals_p; 1887d79b6df6SOr Gerlitz struct mlx5_fields *f; 1888d79b6df6SOr Gerlitz u8 cmd, field_bsize; 1889e3ca4e05SOr Gerlitz u32 s_mask, a_mask; 1890d79b6df6SOr Gerlitz unsigned long mask; 18912b64bebaSOr Gerlitz __be32 mask_be32; 18922b64bebaSOr Gerlitz __be16 mask_be16; 1893d79b6df6SOr Gerlitz void *action; 1894d79b6df6SOr Gerlitz 1895d79b6df6SOr Gerlitz set_masks = &masks[TCA_PEDIT_KEY_EX_CMD_SET]; 1896d79b6df6SOr Gerlitz add_masks = &masks[TCA_PEDIT_KEY_EX_CMD_ADD]; 1897d79b6df6SOr Gerlitz set_vals = &vals[TCA_PEDIT_KEY_EX_CMD_SET]; 1898d79b6df6SOr Gerlitz add_vals = &vals[TCA_PEDIT_KEY_EX_CMD_ADD]; 1899d79b6df6SOr Gerlitz 1900d79b6df6SOr Gerlitz action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto); 1901d79b6df6SOr Gerlitz action = parse_attr->mod_hdr_actions; 1902d79b6df6SOr Gerlitz max_actions = parse_attr->num_mod_hdr_actions; 1903d79b6df6SOr Gerlitz nactions = 0; 1904d79b6df6SOr Gerlitz 1905d79b6df6SOr Gerlitz for (i = 0; i < ARRAY_SIZE(fields); i++) { 1906d79b6df6SOr Gerlitz f = &fields[i]; 1907d79b6df6SOr Gerlitz /* avoid seeing bits set from previous iterations */ 1908e3ca4e05SOr Gerlitz s_mask = 0; 1909e3ca4e05SOr Gerlitz a_mask = 0; 1910d79b6df6SOr Gerlitz 1911d79b6df6SOr Gerlitz s_masks_p = (void *)set_masks + f->offset; 1912d79b6df6SOr Gerlitz a_masks_p = (void *)add_masks + f->offset; 1913d79b6df6SOr Gerlitz 1914d79b6df6SOr Gerlitz memcpy(&s_mask, s_masks_p, f->size); 1915d79b6df6SOr Gerlitz memcpy(&a_mask, a_masks_p, f->size); 1916d79b6df6SOr Gerlitz 1917d79b6df6SOr Gerlitz if (!s_mask && !a_mask) /* nothing to offload here */ 1918d79b6df6SOr Gerlitz continue; 1919d79b6df6SOr Gerlitz 1920d79b6df6SOr Gerlitz if (s_mask && a_mask) { 1921e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 1922e98bedf5SEli Britstein "can't set and add to the same HW field"); 1923d79b6df6SOr Gerlitz printk(KERN_WARNING "mlx5: can't set and add to the same HW field (%x)\n", f->field); 1924d79b6df6SOr Gerlitz return -EOPNOTSUPP; 1925d79b6df6SOr Gerlitz } 1926d79b6df6SOr Gerlitz 1927d79b6df6SOr Gerlitz if (nactions == max_actions) { 1928e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 1929e98bedf5SEli Britstein "too many pedit actions, can't offload"); 1930d79b6df6SOr Gerlitz printk(KERN_WARNING "mlx5: parsed %d pedit actions, can't do more\n", nactions); 1931d79b6df6SOr Gerlitz return -EOPNOTSUPP; 1932d79b6df6SOr Gerlitz } 1933d79b6df6SOr Gerlitz 1934d79b6df6SOr Gerlitz if (s_mask) { 1935d79b6df6SOr Gerlitz cmd = MLX5_ACTION_TYPE_SET; 1936d79b6df6SOr Gerlitz mask = s_mask; 1937d79b6df6SOr Gerlitz vals_p = (void *)set_vals + f->offset; 1938d79b6df6SOr Gerlitz /* clear to denote we consumed this field */ 1939d79b6df6SOr Gerlitz memset(s_masks_p, 0, f->size); 1940d79b6df6SOr Gerlitz } else { 1941d79b6df6SOr Gerlitz cmd = MLX5_ACTION_TYPE_ADD; 1942d79b6df6SOr Gerlitz mask = a_mask; 1943d79b6df6SOr Gerlitz vals_p = (void *)add_vals + f->offset; 1944d79b6df6SOr Gerlitz /* clear to denote we consumed this field */ 1945d79b6df6SOr Gerlitz memset(a_masks_p, 0, f->size); 1946d79b6df6SOr Gerlitz } 1947d79b6df6SOr Gerlitz 1948d79b6df6SOr Gerlitz field_bsize = f->size * BITS_PER_BYTE; 1949e3ca4e05SOr Gerlitz 19502b64bebaSOr Gerlitz if (field_bsize == 32) { 19512b64bebaSOr Gerlitz mask_be32 = *(__be32 *)&mask; 19522b64bebaSOr Gerlitz mask = (__force unsigned long)cpu_to_le32(be32_to_cpu(mask_be32)); 19532b64bebaSOr Gerlitz } else if (field_bsize == 16) { 19542b64bebaSOr Gerlitz mask_be16 = *(__be16 *)&mask; 19552b64bebaSOr Gerlitz mask = (__force unsigned long)cpu_to_le16(be16_to_cpu(mask_be16)); 19562b64bebaSOr Gerlitz } 19572b64bebaSOr Gerlitz 1958d79b6df6SOr Gerlitz first = find_first_bit(&mask, field_bsize); 19592b64bebaSOr Gerlitz next_z = find_next_zero_bit(&mask, field_bsize, first); 1960d79b6df6SOr Gerlitz last = find_last_bit(&mask, field_bsize); 19612b64bebaSOr Gerlitz if (first < next_z && next_z < last) { 1962e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 1963e98bedf5SEli Britstein "rewrite of few sub-fields isn't supported"); 19642b64bebaSOr Gerlitz printk(KERN_WARNING "mlx5: rewrite of few sub-fields (mask %lx) isn't offloaded\n", 1965d79b6df6SOr Gerlitz mask); 1966d79b6df6SOr Gerlitz return -EOPNOTSUPP; 1967d79b6df6SOr Gerlitz } 1968d79b6df6SOr Gerlitz 1969d79b6df6SOr Gerlitz MLX5_SET(set_action_in, action, action_type, cmd); 1970d79b6df6SOr Gerlitz MLX5_SET(set_action_in, action, field, f->field); 1971d79b6df6SOr Gerlitz 1972d79b6df6SOr Gerlitz if (cmd == MLX5_ACTION_TYPE_SET) { 19732b64bebaSOr Gerlitz MLX5_SET(set_action_in, action, offset, first); 1974d79b6df6SOr Gerlitz /* length is num of bits to be written, zero means length of 32 */ 19752b64bebaSOr Gerlitz MLX5_SET(set_action_in, action, length, (last - first + 1)); 1976d79b6df6SOr Gerlitz } 1977d79b6df6SOr Gerlitz 1978d79b6df6SOr Gerlitz if (field_bsize == 32) 19792b64bebaSOr Gerlitz MLX5_SET(set_action_in, action, data, ntohl(*(__be32 *)vals_p) >> first); 1980d79b6df6SOr Gerlitz else if (field_bsize == 16) 19812b64bebaSOr Gerlitz MLX5_SET(set_action_in, action, data, ntohs(*(__be16 *)vals_p) >> first); 1982d79b6df6SOr Gerlitz else if (field_bsize == 8) 19832b64bebaSOr Gerlitz MLX5_SET(set_action_in, action, data, *(u8 *)vals_p >> first); 1984d79b6df6SOr Gerlitz 1985d79b6df6SOr Gerlitz action += action_size; 1986d79b6df6SOr Gerlitz nactions++; 1987d79b6df6SOr Gerlitz } 1988d79b6df6SOr Gerlitz 1989d79b6df6SOr Gerlitz parse_attr->num_mod_hdr_actions = nactions; 1990d79b6df6SOr Gerlitz return 0; 1991d79b6df6SOr Gerlitz } 1992d79b6df6SOr Gerlitz 1993d79b6df6SOr Gerlitz static int alloc_mod_hdr_actions(struct mlx5e_priv *priv, 1994d79b6df6SOr Gerlitz const struct tc_action *a, int namespace, 1995d79b6df6SOr Gerlitz struct mlx5e_tc_flow_parse_attr *parse_attr) 1996d79b6df6SOr Gerlitz { 1997d79b6df6SOr Gerlitz int nkeys, action_size, max_actions; 1998d79b6df6SOr Gerlitz 1999d79b6df6SOr Gerlitz nkeys = tcf_pedit_nkeys(a); 2000d79b6df6SOr Gerlitz action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto); 2001d79b6df6SOr Gerlitz 2002d79b6df6SOr Gerlitz if (namespace == MLX5_FLOW_NAMESPACE_FDB) /* FDB offloading */ 2003d79b6df6SOr Gerlitz max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev, max_modify_header_actions); 2004d79b6df6SOr Gerlitz else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */ 2005d79b6df6SOr Gerlitz max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, max_modify_header_actions); 2006d79b6df6SOr Gerlitz 2007d79b6df6SOr Gerlitz /* can get up to crazingly 16 HW actions in 32 bits pedit SW key */ 2008d79b6df6SOr Gerlitz max_actions = min(max_actions, nkeys * 16); 2009d79b6df6SOr Gerlitz 2010d79b6df6SOr Gerlitz parse_attr->mod_hdr_actions = kcalloc(max_actions, action_size, GFP_KERNEL); 2011d79b6df6SOr Gerlitz if (!parse_attr->mod_hdr_actions) 2012d79b6df6SOr Gerlitz return -ENOMEM; 2013d79b6df6SOr Gerlitz 2014d79b6df6SOr Gerlitz parse_attr->num_mod_hdr_actions = max_actions; 2015d79b6df6SOr Gerlitz return 0; 2016d79b6df6SOr Gerlitz } 2017d79b6df6SOr Gerlitz 2018d79b6df6SOr Gerlitz static const struct pedit_headers zero_masks = {}; 2019d79b6df6SOr Gerlitz 2020d79b6df6SOr Gerlitz static int parse_tc_pedit_action(struct mlx5e_priv *priv, 2021d79b6df6SOr Gerlitz const struct tc_action *a, int namespace, 2022e98bedf5SEli Britstein struct mlx5e_tc_flow_parse_attr *parse_attr, 2023e98bedf5SEli Britstein struct netlink_ext_ack *extack) 2024d79b6df6SOr Gerlitz { 2025d79b6df6SOr Gerlitz struct pedit_headers masks[__PEDIT_CMD_MAX], vals[__PEDIT_CMD_MAX], *cmd_masks; 2026d79b6df6SOr Gerlitz int nkeys, i, err = -EOPNOTSUPP; 2027d79b6df6SOr Gerlitz u32 mask, val, offset; 2028d79b6df6SOr Gerlitz u8 cmd, htype; 2029d79b6df6SOr Gerlitz 2030d79b6df6SOr Gerlitz nkeys = tcf_pedit_nkeys(a); 2031d79b6df6SOr Gerlitz 2032d79b6df6SOr Gerlitz memset(masks, 0, sizeof(struct pedit_headers) * __PEDIT_CMD_MAX); 2033d79b6df6SOr Gerlitz memset(vals, 0, sizeof(struct pedit_headers) * __PEDIT_CMD_MAX); 2034d79b6df6SOr Gerlitz 2035d79b6df6SOr Gerlitz for (i = 0; i < nkeys; i++) { 2036d79b6df6SOr Gerlitz htype = tcf_pedit_htype(a, i); 2037d79b6df6SOr Gerlitz cmd = tcf_pedit_cmd(a, i); 2038d79b6df6SOr Gerlitz err = -EOPNOTSUPP; /* can't be all optimistic */ 2039d79b6df6SOr Gerlitz 2040d79b6df6SOr Gerlitz if (htype == TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK) { 2041e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2042e98bedf5SEli Britstein "legacy pedit isn't offloaded"); 2043d79b6df6SOr Gerlitz goto out_err; 2044d79b6df6SOr Gerlitz } 2045d79b6df6SOr Gerlitz 2046d79b6df6SOr Gerlitz if (cmd != TCA_PEDIT_KEY_EX_CMD_SET && cmd != TCA_PEDIT_KEY_EX_CMD_ADD) { 2047e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, "pedit cmd isn't offloaded"); 2048d79b6df6SOr Gerlitz goto out_err; 2049d79b6df6SOr Gerlitz } 2050d79b6df6SOr Gerlitz 2051d79b6df6SOr Gerlitz mask = tcf_pedit_mask(a, i); 2052d79b6df6SOr Gerlitz val = tcf_pedit_val(a, i); 2053d79b6df6SOr Gerlitz offset = tcf_pedit_offset(a, i); 2054d79b6df6SOr Gerlitz 2055d79b6df6SOr Gerlitz err = set_pedit_val(htype, ~mask, val, offset, &masks[cmd], &vals[cmd]); 2056d79b6df6SOr Gerlitz if (err) 2057d79b6df6SOr Gerlitz goto out_err; 2058d79b6df6SOr Gerlitz } 2059d79b6df6SOr Gerlitz 2060d79b6df6SOr Gerlitz err = alloc_mod_hdr_actions(priv, a, namespace, parse_attr); 2061d79b6df6SOr Gerlitz if (err) 2062d79b6df6SOr Gerlitz goto out_err; 2063d79b6df6SOr Gerlitz 2064e98bedf5SEli Britstein err = offload_pedit_fields(masks, vals, parse_attr, extack); 2065d79b6df6SOr Gerlitz if (err < 0) 2066d79b6df6SOr Gerlitz goto out_dealloc_parsed_actions; 2067d79b6df6SOr Gerlitz 2068d79b6df6SOr Gerlitz for (cmd = 0; cmd < __PEDIT_CMD_MAX; cmd++) { 2069d79b6df6SOr Gerlitz cmd_masks = &masks[cmd]; 2070d79b6df6SOr Gerlitz if (memcmp(cmd_masks, &zero_masks, sizeof(zero_masks))) { 2071e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2072e98bedf5SEli Britstein "attempt to offload an unsupported field"); 2073b3a433deSOr Gerlitz netdev_warn(priv->netdev, "attempt to offload an unsupported field (cmd %d)\n", cmd); 2074d79b6df6SOr Gerlitz print_hex_dump(KERN_WARNING, "mask: ", DUMP_PREFIX_ADDRESS, 2075d79b6df6SOr Gerlitz 16, 1, cmd_masks, sizeof(zero_masks), true); 2076d79b6df6SOr Gerlitz err = -EOPNOTSUPP; 2077d79b6df6SOr Gerlitz goto out_dealloc_parsed_actions; 2078d79b6df6SOr Gerlitz } 2079d79b6df6SOr Gerlitz } 2080d79b6df6SOr Gerlitz 2081d79b6df6SOr Gerlitz return 0; 2082d79b6df6SOr Gerlitz 2083d79b6df6SOr Gerlitz out_dealloc_parsed_actions: 2084d79b6df6SOr Gerlitz kfree(parse_attr->mod_hdr_actions); 2085d79b6df6SOr Gerlitz out_err: 2086d79b6df6SOr Gerlitz return err; 2087d79b6df6SOr Gerlitz } 2088d79b6df6SOr Gerlitz 2089e98bedf5SEli Britstein static bool csum_offload_supported(struct mlx5e_priv *priv, 2090e98bedf5SEli Britstein u32 action, 2091e98bedf5SEli Britstein u32 update_flags, 2092e98bedf5SEli Britstein struct netlink_ext_ack *extack) 209326c02749SOr Gerlitz { 209426c02749SOr Gerlitz u32 prot_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR | TCA_CSUM_UPDATE_FLAG_TCP | 209526c02749SOr Gerlitz TCA_CSUM_UPDATE_FLAG_UDP; 209626c02749SOr Gerlitz 209726c02749SOr Gerlitz /* The HW recalcs checksums only if re-writing headers */ 209826c02749SOr Gerlitz if (!(action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)) { 2099e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2100e98bedf5SEli Britstein "TC csum action is only offloaded with pedit"); 210126c02749SOr Gerlitz netdev_warn(priv->netdev, 210226c02749SOr Gerlitz "TC csum action is only offloaded with pedit\n"); 210326c02749SOr Gerlitz return false; 210426c02749SOr Gerlitz } 210526c02749SOr Gerlitz 210626c02749SOr Gerlitz if (update_flags & ~prot_flags) { 2107e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2108e98bedf5SEli Britstein "can't offload TC csum action for some header/s"); 210926c02749SOr Gerlitz netdev_warn(priv->netdev, 211026c02749SOr Gerlitz "can't offload TC csum action for some header/s - flags %#x\n", 211126c02749SOr Gerlitz update_flags); 211226c02749SOr Gerlitz return false; 211326c02749SOr Gerlitz } 211426c02749SOr Gerlitz 211526c02749SOr Gerlitz return true; 211626c02749SOr Gerlitz } 211726c02749SOr Gerlitz 2118bdd66ac0SOr Gerlitz static bool modify_header_match_supported(struct mlx5_flow_spec *spec, 2119e98bedf5SEli Britstein struct tcf_exts *exts, 2120e98bedf5SEli Britstein struct netlink_ext_ack *extack) 2121bdd66ac0SOr Gerlitz { 2122bdd66ac0SOr Gerlitz const struct tc_action *a; 2123bdd66ac0SOr Gerlitz bool modify_ip_header; 2124bdd66ac0SOr Gerlitz LIST_HEAD(actions); 2125bdd66ac0SOr Gerlitz u8 htype, ip_proto; 2126bdd66ac0SOr Gerlitz void *headers_v; 2127bdd66ac0SOr Gerlitz u16 ethertype; 2128bdd66ac0SOr Gerlitz int nkeys, i; 2129bdd66ac0SOr Gerlitz 2130bdd66ac0SOr Gerlitz headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 2131bdd66ac0SOr Gerlitz ethertype = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ethertype); 2132bdd66ac0SOr Gerlitz 2133bdd66ac0SOr Gerlitz /* for non-IP we only re-write MACs, so we're okay */ 2134bdd66ac0SOr Gerlitz if (ethertype != ETH_P_IP && ethertype != ETH_P_IPV6) 2135bdd66ac0SOr Gerlitz goto out_ok; 2136bdd66ac0SOr Gerlitz 2137bdd66ac0SOr Gerlitz modify_ip_header = false; 2138244cd96aSCong Wang tcf_exts_for_each_action(i, a, exts) { 2139244cd96aSCong Wang int k; 2140244cd96aSCong Wang 2141bdd66ac0SOr Gerlitz if (!is_tcf_pedit(a)) 2142bdd66ac0SOr Gerlitz continue; 2143bdd66ac0SOr Gerlitz 2144bdd66ac0SOr Gerlitz nkeys = tcf_pedit_nkeys(a); 2145244cd96aSCong Wang for (k = 0; k < nkeys; k++) { 2146244cd96aSCong Wang htype = tcf_pedit_htype(a, k); 2147bdd66ac0SOr Gerlitz if (htype == TCA_PEDIT_KEY_EX_HDR_TYPE_IP4 || 2148bdd66ac0SOr Gerlitz htype == TCA_PEDIT_KEY_EX_HDR_TYPE_IP6) { 2149bdd66ac0SOr Gerlitz modify_ip_header = true; 2150bdd66ac0SOr Gerlitz break; 2151bdd66ac0SOr Gerlitz } 2152bdd66ac0SOr Gerlitz } 2153bdd66ac0SOr Gerlitz } 2154bdd66ac0SOr Gerlitz 2155bdd66ac0SOr Gerlitz ip_proto = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol); 21561ccef350SJianbo Liu if (modify_ip_header && ip_proto != IPPROTO_TCP && 21571ccef350SJianbo Liu ip_proto != IPPROTO_UDP && ip_proto != IPPROTO_ICMP) { 2158e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2159e98bedf5SEli Britstein "can't offload re-write of non TCP/UDP"); 2160bdd66ac0SOr Gerlitz pr_info("can't offload re-write of ip proto %d\n", ip_proto); 2161bdd66ac0SOr Gerlitz return false; 2162bdd66ac0SOr Gerlitz } 2163bdd66ac0SOr Gerlitz 2164bdd66ac0SOr Gerlitz out_ok: 2165bdd66ac0SOr Gerlitz return true; 2166bdd66ac0SOr Gerlitz } 2167bdd66ac0SOr Gerlitz 2168bdd66ac0SOr Gerlitz static bool actions_match_supported(struct mlx5e_priv *priv, 2169bdd66ac0SOr Gerlitz struct tcf_exts *exts, 2170bdd66ac0SOr Gerlitz struct mlx5e_tc_flow_parse_attr *parse_attr, 2171e98bedf5SEli Britstein struct mlx5e_tc_flow *flow, 2172e98bedf5SEli Britstein struct netlink_ext_ack *extack) 2173bdd66ac0SOr Gerlitz { 2174bdd66ac0SOr Gerlitz u32 actions; 2175bdd66ac0SOr Gerlitz 2176bdd66ac0SOr Gerlitz if (flow->flags & MLX5E_TC_FLOW_ESWITCH) 2177bdd66ac0SOr Gerlitz actions = flow->esw_attr->action; 2178bdd66ac0SOr Gerlitz else 2179bdd66ac0SOr Gerlitz actions = flow->nic_attr->action; 2180bdd66ac0SOr Gerlitz 21817e29392eSRoi Dayan if (flow->flags & MLX5E_TC_FLOW_EGRESS && 21827e29392eSRoi Dayan !(actions & MLX5_FLOW_CONTEXT_ACTION_DECAP)) 21837e29392eSRoi Dayan return false; 21847e29392eSRoi Dayan 2185bdd66ac0SOr Gerlitz if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 2186e98bedf5SEli Britstein return modify_header_match_supported(&parse_attr->spec, exts, 2187e98bedf5SEli Britstein extack); 2188bdd66ac0SOr Gerlitz 2189bdd66ac0SOr Gerlitz return true; 2190bdd66ac0SOr Gerlitz } 2191bdd66ac0SOr Gerlitz 21925c65c564SOr Gerlitz static bool same_hw_devs(struct mlx5e_priv *priv, struct mlx5e_priv *peer_priv) 21935c65c564SOr Gerlitz { 21945c65c564SOr Gerlitz struct mlx5_core_dev *fmdev, *pmdev; 2195816f6706SOr Gerlitz u64 fsystem_guid, psystem_guid; 21965c65c564SOr Gerlitz 21975c65c564SOr Gerlitz fmdev = priv->mdev; 21985c65c564SOr Gerlitz pmdev = peer_priv->mdev; 21995c65c564SOr Gerlitz 220059c9d35eSAlaa Hleihel fsystem_guid = mlx5_query_nic_system_image_guid(fmdev); 220159c9d35eSAlaa Hleihel psystem_guid = mlx5_query_nic_system_image_guid(pmdev); 22025c65c564SOr Gerlitz 2203816f6706SOr Gerlitz return (fsystem_guid == psystem_guid); 22045c65c564SOr Gerlitz } 22055c65c564SOr Gerlitz 22065c40348cSOr Gerlitz static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts, 2207aa0cbbaeSOr Gerlitz struct mlx5e_tc_flow_parse_attr *parse_attr, 2208e98bedf5SEli Britstein struct mlx5e_tc_flow *flow, 2209e98bedf5SEli Britstein struct netlink_ext_ack *extack) 2210e3a2b7edSAmir Vadai { 2211aa0cbbaeSOr Gerlitz struct mlx5_nic_flow_attr *attr = flow->nic_attr; 2212e3a2b7edSAmir Vadai const struct tc_action *a; 221322dc13c8SWANG Cong LIST_HEAD(actions); 22141cab1cd7SOr Gerlitz u32 action = 0; 2215244cd96aSCong Wang int err, i; 2216e3a2b7edSAmir Vadai 22173bcc0cecSJiri Pirko if (!tcf_exts_has_actions(exts)) 2218e3a2b7edSAmir Vadai return -EINVAL; 2219e3a2b7edSAmir Vadai 22203bc4b7bfSOr Gerlitz attr->flow_tag = MLX5_FS_DEFAULT_FLOW_TAG; 2221e3a2b7edSAmir Vadai 2222244cd96aSCong Wang tcf_exts_for_each_action(i, a, exts) { 2223e3a2b7edSAmir Vadai if (is_tcf_gact_shot(a)) { 22241cab1cd7SOr Gerlitz action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 2225aad7e08dSAmir Vadai if (MLX5_CAP_FLOWTABLE(priv->mdev, 2226aad7e08dSAmir Vadai flow_table_properties_nic_receive.flow_counter)) 22271cab1cd7SOr Gerlitz action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 2228e3a2b7edSAmir Vadai continue; 2229e3a2b7edSAmir Vadai } 2230e3a2b7edSAmir Vadai 22312f4fe4caSOr Gerlitz if (is_tcf_pedit(a)) { 22322f4fe4caSOr Gerlitz err = parse_tc_pedit_action(priv, a, MLX5_FLOW_NAMESPACE_KERNEL, 2233e98bedf5SEli Britstein parse_attr, extack); 22342f4fe4caSOr Gerlitz if (err) 22352f4fe4caSOr Gerlitz return err; 22362f4fe4caSOr Gerlitz 22371cab1cd7SOr Gerlitz action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | 22382f4fe4caSOr Gerlitz MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 22392f4fe4caSOr Gerlitz continue; 22402f4fe4caSOr Gerlitz } 22412f4fe4caSOr Gerlitz 224226c02749SOr Gerlitz if (is_tcf_csum(a)) { 22431cab1cd7SOr Gerlitz if (csum_offload_supported(priv, action, 2244e98bedf5SEli Britstein tcf_csum_update_flags(a), 2245e98bedf5SEli Britstein extack)) 224626c02749SOr Gerlitz continue; 224726c02749SOr Gerlitz 224826c02749SOr Gerlitz return -EOPNOTSUPP; 224926c02749SOr Gerlitz } 225026c02749SOr Gerlitz 22515c65c564SOr Gerlitz if (is_tcf_mirred_egress_redirect(a)) { 22525c65c564SOr Gerlitz struct net_device *peer_dev = tcf_mirred_dev(a); 22535c65c564SOr Gerlitz 22545c65c564SOr Gerlitz if (priv->netdev->netdev_ops == peer_dev->netdev_ops && 22555c65c564SOr Gerlitz same_hw_devs(priv, netdev_priv(peer_dev))) { 225698b66cb1SEli Britstein parse_attr->mirred_ifindex[0] = peer_dev->ifindex; 22575c65c564SOr Gerlitz flow->flags |= MLX5E_TC_FLOW_HAIRPIN; 22581cab1cd7SOr Gerlitz action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 22595c65c564SOr Gerlitz MLX5_FLOW_CONTEXT_ACTION_COUNT; 22605c65c564SOr Gerlitz } else { 2261e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2262e98bedf5SEli Britstein "device is not on same HW, can't offload"); 22635c65c564SOr Gerlitz netdev_warn(priv->netdev, "device %s not on same HW, can't offload\n", 22645c65c564SOr Gerlitz peer_dev->name); 22655c65c564SOr Gerlitz return -EINVAL; 22665c65c564SOr Gerlitz } 22675c65c564SOr Gerlitz continue; 22685c65c564SOr Gerlitz } 22695c65c564SOr Gerlitz 2270e3a2b7edSAmir Vadai if (is_tcf_skbedit_mark(a)) { 2271e3a2b7edSAmir Vadai u32 mark = tcf_skbedit_mark(a); 2272e3a2b7edSAmir Vadai 2273e3a2b7edSAmir Vadai if (mark & ~MLX5E_TC_FLOW_ID_MASK) { 2274e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2275e98bedf5SEli Britstein "Bad flow mark - only 16 bit is supported"); 2276e3a2b7edSAmir Vadai return -EINVAL; 2277e3a2b7edSAmir Vadai } 2278e3a2b7edSAmir Vadai 22793bc4b7bfSOr Gerlitz attr->flow_tag = mark; 22801cab1cd7SOr Gerlitz action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2281e3a2b7edSAmir Vadai continue; 2282e3a2b7edSAmir Vadai } 2283e3a2b7edSAmir Vadai 2284e3a2b7edSAmir Vadai return -EINVAL; 2285e3a2b7edSAmir Vadai } 2286e3a2b7edSAmir Vadai 22871cab1cd7SOr Gerlitz attr->action = action; 2288e98bedf5SEli Britstein if (!actions_match_supported(priv, exts, parse_attr, flow, extack)) 2289bdd66ac0SOr Gerlitz return -EOPNOTSUPP; 2290bdd66ac0SOr Gerlitz 2291e3a2b7edSAmir Vadai return 0; 2292e3a2b7edSAmir Vadai } 2293e3a2b7edSAmir Vadai 229476f7444dSOr Gerlitz static inline int cmp_encap_info(struct ip_tunnel_key *a, 229576f7444dSOr Gerlitz struct ip_tunnel_key *b) 2296a54e20b4SHadar Hen Zion { 2297a54e20b4SHadar Hen Zion return memcmp(a, b, sizeof(*a)); 2298a54e20b4SHadar Hen Zion } 2299a54e20b4SHadar Hen Zion 230076f7444dSOr Gerlitz static inline int hash_encap_info(struct ip_tunnel_key *key) 2301a54e20b4SHadar Hen Zion { 230276f7444dSOr Gerlitz return jhash(key, sizeof(*key), 0); 2303a54e20b4SHadar Hen Zion } 2304a54e20b4SHadar Hen Zion 2305a54e20b4SHadar Hen Zion 2306b1d90e6bSRabie Loulou static bool is_merged_eswitch_dev(struct mlx5e_priv *priv, 2307b1d90e6bSRabie Loulou struct net_device *peer_netdev) 2308b1d90e6bSRabie Loulou { 2309b1d90e6bSRabie Loulou struct mlx5e_priv *peer_priv; 2310b1d90e6bSRabie Loulou 2311b1d90e6bSRabie Loulou peer_priv = netdev_priv(peer_netdev); 2312b1d90e6bSRabie Loulou 2313b1d90e6bSRabie Loulou return (MLX5_CAP_ESW(priv->mdev, merged_eswitch) && 2314b1d90e6bSRabie Loulou (priv->netdev->netdev_ops == peer_netdev->netdev_ops) && 2315b1d90e6bSRabie Loulou same_hw_devs(priv, peer_priv) && 2316b1d90e6bSRabie Loulou MLX5_VPORT_MANAGER(peer_priv->mdev) && 2317b1d90e6bSRabie Loulou (peer_priv->mdev->priv.eswitch->mode == SRIOV_OFFLOADS)); 2318b1d90e6bSRabie Loulou } 2319b1d90e6bSRabie Loulou 2320ce99f6b9SOr Gerlitz 232154c177caSOz Shlomo 2322a54e20b4SHadar Hen Zion static int mlx5e_attach_encap(struct mlx5e_priv *priv, 2323a54e20b4SHadar Hen Zion struct ip_tunnel_info *tun_info, 2324a54e20b4SHadar Hen Zion struct net_device *mirred_dev, 232545247bf2SOr Gerlitz struct net_device **encap_dev, 2326e98bedf5SEli Britstein struct mlx5e_tc_flow *flow, 2327e98bedf5SEli Britstein struct netlink_ext_ack *extack) 232803a9d11eSOr Gerlitz { 2329a54e20b4SHadar Hen Zion struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 2330a54e20b4SHadar Hen Zion unsigned short family = ip_tunnel_info_af(tun_info); 233145247bf2SOr Gerlitz struct mlx5_esw_flow_attr *attr = flow->esw_attr; 2332a54e20b4SHadar Hen Zion struct ip_tunnel_key *key = &tun_info->key; 2333c1ae1152SOr Gerlitz struct mlx5e_encap_entry *e; 2334a54e20b4SHadar Hen Zion uintptr_t hash_key; 2335a54e20b4SHadar Hen Zion bool found = false; 233654c177caSOz Shlomo int err = 0; 2337a54e20b4SHadar Hen Zion 233876f7444dSOr Gerlitz hash_key = hash_encap_info(key); 2339a54e20b4SHadar Hen Zion 2340a54e20b4SHadar Hen Zion hash_for_each_possible_rcu(esw->offloads.encap_tbl, e, 2341a54e20b4SHadar Hen Zion encap_hlist, hash_key) { 234276f7444dSOr Gerlitz if (!cmp_encap_info(&e->tun_info.key, key)) { 2343a54e20b4SHadar Hen Zion found = true; 2344a54e20b4SHadar Hen Zion break; 2345a54e20b4SHadar Hen Zion } 2346a54e20b4SHadar Hen Zion } 2347a54e20b4SHadar Hen Zion 2348b2812089SVlad Buslov /* must verify if encap is valid or not */ 234945247bf2SOr Gerlitz if (found) 235045247bf2SOr Gerlitz goto attach_flow; 2351a54e20b4SHadar Hen Zion 2352a54e20b4SHadar Hen Zion e = kzalloc(sizeof(*e), GFP_KERNEL); 2353a54e20b4SHadar Hen Zion if (!e) 2354a54e20b4SHadar Hen Zion return -ENOMEM; 2355a54e20b4SHadar Hen Zion 235676f7444dSOr Gerlitz e->tun_info = *tun_info; 2357101f4de9SOz Shlomo err = mlx5e_tc_tun_init_encap_attr(mirred_dev, priv, e, extack); 235854c177caSOz Shlomo if (err) 235954c177caSOz Shlomo goto out_err; 236054c177caSOz Shlomo 2361a54e20b4SHadar Hen Zion INIT_LIST_HEAD(&e->flows); 2362a54e20b4SHadar Hen Zion 2363ce99f6b9SOr Gerlitz if (family == AF_INET) 2364101f4de9SOz Shlomo err = mlx5e_tc_tun_create_header_ipv4(priv, mirred_dev, e); 2365ce99f6b9SOr Gerlitz else if (family == AF_INET6) 2366101f4de9SOz Shlomo err = mlx5e_tc_tun_create_header_ipv6(priv, mirred_dev, e); 2367ce99f6b9SOr Gerlitz 2368232c0013SHadar Hen Zion if (err && err != -EAGAIN) 2369a54e20b4SHadar Hen Zion goto out_err; 2370a54e20b4SHadar Hen Zion 2371a54e20b4SHadar Hen Zion hash_add_rcu(esw->offloads.encap_tbl, &e->encap_hlist, hash_key); 2372a54e20b4SHadar Hen Zion 237345247bf2SOr Gerlitz attach_flow: 237479baaec7SEli Britstein list_add(&flow->encaps[0].list, &e->flows); 237579baaec7SEli Britstein flow->encaps[0].index = 0; 237645247bf2SOr Gerlitz *encap_dev = e->out_dev; 2377232c0013SHadar Hen Zion if (e->flags & MLX5_ENCAP_ENTRY_VALID) 237845247bf2SOr Gerlitz attr->encap_id = e->encap_id; 2379b2812089SVlad Buslov else 2380b2812089SVlad Buslov err = -EAGAIN; 238145247bf2SOr Gerlitz 2382232c0013SHadar Hen Zion return err; 2383a54e20b4SHadar Hen Zion 2384a54e20b4SHadar Hen Zion out_err: 2385a54e20b4SHadar Hen Zion kfree(e); 2386a54e20b4SHadar Hen Zion return err; 2387a54e20b4SHadar Hen Zion } 2388a54e20b4SHadar Hen Zion 23891482bd3dSJianbo Liu static int parse_tc_vlan_action(struct mlx5e_priv *priv, 23901482bd3dSJianbo Liu const struct tc_action *a, 23911482bd3dSJianbo Liu struct mlx5_esw_flow_attr *attr, 23921482bd3dSJianbo Liu u32 *action) 23931482bd3dSJianbo Liu { 2394cc495188SJianbo Liu u8 vlan_idx = attr->total_vlan; 2395cc495188SJianbo Liu 2396cc495188SJianbo Liu if (vlan_idx >= MLX5_FS_VLAN_DEPTH) 23971482bd3dSJianbo Liu return -EOPNOTSUPP; 2398cc495188SJianbo Liu 2399cc495188SJianbo Liu if (tcf_vlan_action(a) == TCA_VLAN_ACT_POP) { 2400cc495188SJianbo Liu if (vlan_idx) { 2401cc495188SJianbo Liu if (!mlx5_eswitch_vlan_actions_supported(priv->mdev, 2402cc495188SJianbo Liu MLX5_FS_VLAN_DEPTH)) 2403cc495188SJianbo Liu return -EOPNOTSUPP; 2404cc495188SJianbo Liu 2405cc495188SJianbo Liu *action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2; 2406cc495188SJianbo Liu } else { 2407cc495188SJianbo Liu *action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP; 2408cc495188SJianbo Liu } 2409cc495188SJianbo Liu } else if (tcf_vlan_action(a) == TCA_VLAN_ACT_PUSH) { 2410cc495188SJianbo Liu attr->vlan_vid[vlan_idx] = tcf_vlan_push_vid(a); 2411cc495188SJianbo Liu attr->vlan_prio[vlan_idx] = tcf_vlan_push_prio(a); 2412cc495188SJianbo Liu attr->vlan_proto[vlan_idx] = tcf_vlan_push_proto(a); 2413cc495188SJianbo Liu if (!attr->vlan_proto[vlan_idx]) 2414cc495188SJianbo Liu attr->vlan_proto[vlan_idx] = htons(ETH_P_8021Q); 2415cc495188SJianbo Liu 2416cc495188SJianbo Liu if (vlan_idx) { 2417cc495188SJianbo Liu if (!mlx5_eswitch_vlan_actions_supported(priv->mdev, 2418cc495188SJianbo Liu MLX5_FS_VLAN_DEPTH)) 2419cc495188SJianbo Liu return -EOPNOTSUPP; 2420cc495188SJianbo Liu 2421cc495188SJianbo Liu *action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2; 2422cc495188SJianbo Liu } else { 2423cc495188SJianbo Liu if (!mlx5_eswitch_vlan_actions_supported(priv->mdev, 1) && 2424cc495188SJianbo Liu (tcf_vlan_push_proto(a) != htons(ETH_P_8021Q) || 2425cc495188SJianbo Liu tcf_vlan_push_prio(a))) 2426cc495188SJianbo Liu return -EOPNOTSUPP; 2427cc495188SJianbo Liu 2428cc495188SJianbo Liu *action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH; 24291482bd3dSJianbo Liu } 24301482bd3dSJianbo Liu } else { /* action is TCA_VLAN_ACT_MODIFY */ 24311482bd3dSJianbo Liu return -EOPNOTSUPP; 24321482bd3dSJianbo Liu } 24331482bd3dSJianbo Liu 2434cc495188SJianbo Liu attr->total_vlan = vlan_idx + 1; 2435cc495188SJianbo Liu 24361482bd3dSJianbo Liu return 0; 24371482bd3dSJianbo Liu } 24381482bd3dSJianbo Liu 2439a54e20b4SHadar Hen Zion static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts, 2440d7e75a32SOr Gerlitz struct mlx5e_tc_flow_parse_attr *parse_attr, 2441e98bedf5SEli Britstein struct mlx5e_tc_flow *flow, 2442e98bedf5SEli Britstein struct netlink_ext_ack *extack) 2443a54e20b4SHadar Hen Zion { 2444bf07aa73SPaul Blakey struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 2445ecf5bb79SOr Gerlitz struct mlx5_esw_flow_attr *attr = flow->esw_attr; 24461d447a39SSaeed Mahameed struct mlx5e_rep_priv *rpriv = priv->ppriv; 2447a54e20b4SHadar Hen Zion struct ip_tunnel_info *info = NULL; 244803a9d11eSOr Gerlitz const struct tc_action *a; 244922dc13c8SWANG Cong LIST_HEAD(actions); 2450a54e20b4SHadar Hen Zion bool encap = false; 24511cab1cd7SOr Gerlitz u32 action = 0; 2452244cd96aSCong Wang int err, i; 245303a9d11eSOr Gerlitz 24543bcc0cecSJiri Pirko if (!tcf_exts_has_actions(exts)) 245503a9d11eSOr Gerlitz return -EINVAL; 245603a9d11eSOr Gerlitz 24571d447a39SSaeed Mahameed attr->in_rep = rpriv->rep; 245810ff5359SShahar Klein attr->in_mdev = priv->mdev; 245903a9d11eSOr Gerlitz 2460244cd96aSCong Wang tcf_exts_for_each_action(i, a, exts) { 246103a9d11eSOr Gerlitz if (is_tcf_gact_shot(a)) { 24621cab1cd7SOr Gerlitz action |= MLX5_FLOW_CONTEXT_ACTION_DROP | 246303a9d11eSOr Gerlitz MLX5_FLOW_CONTEXT_ACTION_COUNT; 246403a9d11eSOr Gerlitz continue; 246503a9d11eSOr Gerlitz } 246603a9d11eSOr Gerlitz 2467d7e75a32SOr Gerlitz if (is_tcf_pedit(a)) { 2468d7e75a32SOr Gerlitz err = parse_tc_pedit_action(priv, a, MLX5_FLOW_NAMESPACE_FDB, 2469e98bedf5SEli Britstein parse_attr, extack); 2470d7e75a32SOr Gerlitz if (err) 2471d7e75a32SOr Gerlitz return err; 2472d7e75a32SOr Gerlitz 24731cab1cd7SOr Gerlitz action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 2474e85e02baSEli Britstein attr->split_count = attr->out_count; 2475d7e75a32SOr Gerlitz continue; 2476d7e75a32SOr Gerlitz } 2477d7e75a32SOr Gerlitz 247826c02749SOr Gerlitz if (is_tcf_csum(a)) { 24791cab1cd7SOr Gerlitz if (csum_offload_supported(priv, action, 2480e98bedf5SEli Britstein tcf_csum_update_flags(a), 2481e98bedf5SEli Britstein extack)) 248226c02749SOr Gerlitz continue; 248326c02749SOr Gerlitz 248426c02749SOr Gerlitz return -EOPNOTSUPP; 248526c02749SOr Gerlitz } 248626c02749SOr Gerlitz 2487592d3651SChris Mi if (is_tcf_mirred_egress_redirect(a) || is_tcf_mirred_egress_mirror(a)) { 248803a9d11eSOr Gerlitz struct mlx5e_priv *out_priv; 2489592d3651SChris Mi struct net_device *out_dev; 249003a9d11eSOr Gerlitz 24919f8a739eSCong Wang out_dev = tcf_mirred_dev(a); 2492ef381359SOz Shlomo if (!out_dev) { 2493ef381359SOz Shlomo /* out_dev is NULL when filters with 2494ef381359SOz Shlomo * non-existing mirred device are replayed to 2495ef381359SOz Shlomo * the driver. 2496ef381359SOz Shlomo */ 2497ef381359SOz Shlomo return -EINVAL; 2498ef381359SOz Shlomo } 249903a9d11eSOr Gerlitz 2500592d3651SChris Mi if (attr->out_count >= MLX5_MAX_FLOW_FWD_VPORTS) { 2501e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2502e98bedf5SEli Britstein "can't support more output ports, can't offload forwarding"); 2503592d3651SChris Mi pr_err("can't support more than %d output ports, can't offload forwarding\n", 2504592d3651SChris Mi attr->out_count); 2505592d3651SChris Mi return -EOPNOTSUPP; 2506592d3651SChris Mi } 2507592d3651SChris Mi 2508f493f155SEli Britstein action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 2509f493f155SEli Britstein MLX5_FLOW_CONTEXT_ACTION_COUNT; 2510a54e20b4SHadar Hen Zion if (switchdev_port_same_parent_id(priv->netdev, 2511b1d90e6bSRabie Loulou out_dev) || 2512b1d90e6bSRabie Loulou is_merged_eswitch_dev(priv, out_dev)) { 251303a9d11eSOr Gerlitz out_priv = netdev_priv(out_dev); 25141d447a39SSaeed Mahameed rpriv = out_priv->ppriv; 2515df65a573SEli Britstein attr->dests[attr->out_count].rep = rpriv->rep; 2516df65a573SEli Britstein attr->dests[attr->out_count].mdev = out_priv->mdev; 2517df65a573SEli Britstein attr->out_count++; 2518a54e20b4SHadar Hen Zion } else if (encap) { 251998b66cb1SEli Britstein parse_attr->mirred_ifindex[0] = out_dev->ifindex; 252098b66cb1SEli Britstein parse_attr->tun_info[0] = *info; 25213c37745eSOr Gerlitz attr->parse_attr = parse_attr; 2522f493f155SEli Britstein attr->dests[attr->out_count].flags |= 2523f493f155SEli Britstein MLX5_ESW_DEST_ENCAP; 25241cc26d74SEli Britstein attr->out_count++; 2525df65a573SEli Britstein /* attr->dests[].rep is resolved when we 2526df65a573SEli Britstein * handle encap 2527df65a573SEli Britstein */ 2528ef381359SOz Shlomo } else if (parse_attr->filter_dev != priv->netdev) { 2529ef381359SOz Shlomo /* All mlx5 devices are called to configure 2530ef381359SOz Shlomo * high level device filters. Therefore, the 2531ef381359SOz Shlomo * *attempt* to install a filter on invalid 2532ef381359SOz Shlomo * eswitch should not trigger an explicit error 2533ef381359SOz Shlomo */ 2534ef381359SOz Shlomo return -EINVAL; 2535a54e20b4SHadar Hen Zion } else { 2536e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2537e98bedf5SEli Britstein "devices are not on same switch HW, can't offload forwarding"); 2538a54e20b4SHadar Hen Zion pr_err("devices %s %s not on same switch HW, can't offload forwarding\n", 2539a54e20b4SHadar Hen Zion priv->netdev->name, out_dev->name); 2540a54e20b4SHadar Hen Zion return -EINVAL; 2541a54e20b4SHadar Hen Zion } 2542a54e20b4SHadar Hen Zion continue; 2543a54e20b4SHadar Hen Zion } 2544a54e20b4SHadar Hen Zion 2545a54e20b4SHadar Hen Zion if (is_tcf_tunnel_set(a)) { 2546a54e20b4SHadar Hen Zion info = tcf_tunnel_info(a); 2547a54e20b4SHadar Hen Zion if (info) 2548a54e20b4SHadar Hen Zion encap = true; 2549a54e20b4SHadar Hen Zion else 2550a54e20b4SHadar Hen Zion return -EOPNOTSUPP; 255103a9d11eSOr Gerlitz continue; 255203a9d11eSOr Gerlitz } 255303a9d11eSOr Gerlitz 25548b32580dSOr Gerlitz if (is_tcf_vlan(a)) { 25551482bd3dSJianbo Liu err = parse_tc_vlan_action(priv, a, attr, &action); 25561482bd3dSJianbo Liu 25571482bd3dSJianbo Liu if (err) 25581482bd3dSJianbo Liu return err; 25591482bd3dSJianbo Liu 2560e85e02baSEli Britstein attr->split_count = attr->out_count; 25618b32580dSOr Gerlitz continue; 25628b32580dSOr Gerlitz } 25638b32580dSOr Gerlitz 2564bbd00f7eSHadar Hen Zion if (is_tcf_tunnel_release(a)) { 25651cab1cd7SOr Gerlitz action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 2566bbd00f7eSHadar Hen Zion continue; 2567bbd00f7eSHadar Hen Zion } 2568bbd00f7eSHadar Hen Zion 2569bf07aa73SPaul Blakey if (is_tcf_gact_goto_chain(a)) { 2570bf07aa73SPaul Blakey u32 dest_chain = tcf_gact_goto_chain_index(a); 2571bf07aa73SPaul Blakey u32 max_chain = mlx5_eswitch_get_chain_range(esw); 2572bf07aa73SPaul Blakey 2573bf07aa73SPaul Blakey if (dest_chain <= attr->chain) { 2574bf07aa73SPaul Blakey NL_SET_ERR_MSG(extack, "Goto earlier chain isn't supported"); 2575bf07aa73SPaul Blakey return -EOPNOTSUPP; 2576bf07aa73SPaul Blakey } 2577bf07aa73SPaul Blakey if (dest_chain > max_chain) { 2578bf07aa73SPaul Blakey NL_SET_ERR_MSG(extack, "Requested destination chain is out of supported range"); 2579bf07aa73SPaul Blakey return -EOPNOTSUPP; 2580bf07aa73SPaul Blakey } 2581bf07aa73SPaul Blakey action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 2582bf07aa73SPaul Blakey MLX5_FLOW_CONTEXT_ACTION_COUNT; 2583bf07aa73SPaul Blakey attr->dest_chain = dest_chain; 2584bf07aa73SPaul Blakey 2585bf07aa73SPaul Blakey continue; 2586bf07aa73SPaul Blakey } 2587bf07aa73SPaul Blakey 258803a9d11eSOr Gerlitz return -EINVAL; 258903a9d11eSOr Gerlitz } 2590bdd66ac0SOr Gerlitz 25911cab1cd7SOr Gerlitz attr->action = action; 2592e98bedf5SEli Britstein if (!actions_match_supported(priv, exts, parse_attr, flow, extack)) 2593bdd66ac0SOr Gerlitz return -EOPNOTSUPP; 2594bdd66ac0SOr Gerlitz 2595e85e02baSEli Britstein if (attr->split_count > 0 && !mlx5_esw_has_fwd_fdb(priv->mdev)) { 2596e98bedf5SEli Britstein NL_SET_ERR_MSG_MOD(extack, 2597e98bedf5SEli Britstein "current firmware doesn't support split rule for port mirroring"); 2598592d3651SChris Mi netdev_warn_once(priv->netdev, "current firmware doesn't support split rule for port mirroring\n"); 2599592d3651SChris Mi return -EOPNOTSUPP; 2600592d3651SChris Mi } 2601592d3651SChris Mi 260231c8eba5SOr Gerlitz return 0; 260303a9d11eSOr Gerlitz } 260403a9d11eSOr Gerlitz 26055dbe906fSPaul Blakey static void get_flags(int flags, u16 *flow_flags) 260660bd4af8SOr Gerlitz { 26075dbe906fSPaul Blakey u16 __flow_flags = 0; 260860bd4af8SOr Gerlitz 260960bd4af8SOr Gerlitz if (flags & MLX5E_TC_INGRESS) 261060bd4af8SOr Gerlitz __flow_flags |= MLX5E_TC_FLOW_INGRESS; 261160bd4af8SOr Gerlitz if (flags & MLX5E_TC_EGRESS) 261260bd4af8SOr Gerlitz __flow_flags |= MLX5E_TC_FLOW_EGRESS; 261360bd4af8SOr Gerlitz 261460bd4af8SOr Gerlitz *flow_flags = __flow_flags; 261560bd4af8SOr Gerlitz } 261660bd4af8SOr Gerlitz 261705866c82SOr Gerlitz static const struct rhashtable_params tc_ht_params = { 261805866c82SOr Gerlitz .head_offset = offsetof(struct mlx5e_tc_flow, node), 261905866c82SOr Gerlitz .key_offset = offsetof(struct mlx5e_tc_flow, cookie), 262005866c82SOr Gerlitz .key_len = sizeof(((struct mlx5e_tc_flow *)0)->cookie), 262105866c82SOr Gerlitz .automatic_shrinking = true, 262205866c82SOr Gerlitz }; 262305866c82SOr Gerlitz 262405866c82SOr Gerlitz static struct rhashtable *get_tc_ht(struct mlx5e_priv *priv) 262505866c82SOr Gerlitz { 2626655dc3d2SOr Gerlitz struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 2627655dc3d2SOr Gerlitz struct mlx5e_rep_priv *uplink_rpriv; 2628655dc3d2SOr Gerlitz 2629655dc3d2SOr Gerlitz if (MLX5_VPORT_MANAGER(priv->mdev) && esw->mode == SRIOV_OFFLOADS) { 2630655dc3d2SOr Gerlitz uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); 2631ec1366c2SOz Shlomo return &uplink_rpriv->uplink_priv.tc_ht; 2632655dc3d2SOr Gerlitz } else 263305866c82SOr Gerlitz return &priv->fs.tc.ht; 263405866c82SOr Gerlitz } 263505866c82SOr Gerlitz 2636a88780a9SRoi Dayan static int 2637a88780a9SRoi Dayan mlx5e_alloc_flow(struct mlx5e_priv *priv, int attr_size, 26385dbe906fSPaul Blakey struct tc_cls_flower_offload *f, u16 flow_flags, 2639a88780a9SRoi Dayan struct mlx5e_tc_flow_parse_attr **__parse_attr, 2640a88780a9SRoi Dayan struct mlx5e_tc_flow **__flow) 2641e3a2b7edSAmir Vadai { 264217091853SOr Gerlitz struct mlx5e_tc_flow_parse_attr *parse_attr; 26433bc4b7bfSOr Gerlitz struct mlx5e_tc_flow *flow; 2644a88780a9SRoi Dayan int err; 2645776b12b6SOr Gerlitz 264665ba8fb7SOr Gerlitz flow = kzalloc(sizeof(*flow) + attr_size, GFP_KERNEL); 26471b9a07eeSLeon Romanovsky parse_attr = kvzalloc(sizeof(*parse_attr), GFP_KERNEL); 264817091853SOr Gerlitz if (!parse_attr || !flow) { 2649e3a2b7edSAmir Vadai err = -ENOMEM; 2650e3a2b7edSAmir Vadai goto err_free; 2651e3a2b7edSAmir Vadai } 2652e3a2b7edSAmir Vadai 2653e3a2b7edSAmir Vadai flow->cookie = f->cookie; 265465ba8fb7SOr Gerlitz flow->flags = flow_flags; 2655655dc3d2SOr Gerlitz flow->priv = priv; 2656e3a2b7edSAmir Vadai 2657a88780a9SRoi Dayan *__flow = flow; 2658a88780a9SRoi Dayan *__parse_attr = parse_attr; 2659a88780a9SRoi Dayan 2660a88780a9SRoi Dayan return 0; 2661a88780a9SRoi Dayan 2662a88780a9SRoi Dayan err_free: 2663a88780a9SRoi Dayan kfree(flow); 2664a88780a9SRoi Dayan kvfree(parse_attr); 2665a88780a9SRoi Dayan return err; 2666adb4c123SOr Gerlitz } 2667adb4c123SOr Gerlitz 2668a88780a9SRoi Dayan static int 2669a88780a9SRoi Dayan mlx5e_add_fdb_flow(struct mlx5e_priv *priv, 2670a88780a9SRoi Dayan struct tc_cls_flower_offload *f, 26715dbe906fSPaul Blakey u16 flow_flags, 2672d11afc26SOz Shlomo struct net_device *filter_dev, 2673a88780a9SRoi Dayan struct mlx5e_tc_flow **__flow) 2674a88780a9SRoi Dayan { 2675a88780a9SRoi Dayan struct netlink_ext_ack *extack = f->common.extack; 2676a88780a9SRoi Dayan struct mlx5e_tc_flow_parse_attr *parse_attr; 2677a88780a9SRoi Dayan struct mlx5e_tc_flow *flow; 2678a88780a9SRoi Dayan int attr_size, err; 2679a88780a9SRoi Dayan 2680a88780a9SRoi Dayan flow_flags |= MLX5E_TC_FLOW_ESWITCH; 2681a88780a9SRoi Dayan attr_size = sizeof(struct mlx5_esw_flow_attr); 2682a88780a9SRoi Dayan err = mlx5e_alloc_flow(priv, attr_size, f, flow_flags, 2683a88780a9SRoi Dayan &parse_attr, &flow); 2684a88780a9SRoi Dayan if (err) 2685a88780a9SRoi Dayan goto out; 2686d11afc26SOz Shlomo parse_attr->filter_dev = filter_dev; 2687d11afc26SOz Shlomo flow->esw_attr->parse_attr = parse_attr; 268854c177caSOz Shlomo err = parse_cls_flower(flow->priv, flow, &parse_attr->spec, 268954c177caSOz Shlomo f, filter_dev); 2690d11afc26SOz Shlomo if (err) 2691d11afc26SOz Shlomo goto err_free; 2692a88780a9SRoi Dayan 2693bf07aa73SPaul Blakey flow->esw_attr->chain = f->common.chain_index; 2694bf07aa73SPaul Blakey flow->esw_attr->prio = TC_H_MAJ(f->common.prio) >> 16; 2695a88780a9SRoi Dayan err = parse_tc_fdb_actions(priv, f->exts, parse_attr, flow, extack); 2696a88780a9SRoi Dayan if (err) 2697a88780a9SRoi Dayan goto err_free; 2698a88780a9SRoi Dayan 2699a88780a9SRoi Dayan err = mlx5e_tc_add_fdb_flow(priv, parse_attr, flow, extack); 27005dbe906fSPaul Blakey if (err) 2701aa0cbbaeSOr Gerlitz goto err_free; 27025c40348cSOr Gerlitz 2703a88780a9SRoi Dayan *__flow = flow; 2704af1607c3SJianbo Liu 2705a88780a9SRoi Dayan return 0; 2706e3a2b7edSAmir Vadai 2707e3a2b7edSAmir Vadai err_free: 2708232c0013SHadar Hen Zion kfree(flow); 2709a88780a9SRoi Dayan kvfree(parse_attr); 2710a88780a9SRoi Dayan out: 2711a88780a9SRoi Dayan return err; 2712a88780a9SRoi Dayan } 2713a88780a9SRoi Dayan 2714a88780a9SRoi Dayan static int 2715a88780a9SRoi Dayan mlx5e_add_nic_flow(struct mlx5e_priv *priv, 2716a88780a9SRoi Dayan struct tc_cls_flower_offload *f, 27175dbe906fSPaul Blakey u16 flow_flags, 2718d11afc26SOz Shlomo struct net_device *filter_dev, 2719a88780a9SRoi Dayan struct mlx5e_tc_flow **__flow) 2720a88780a9SRoi Dayan { 2721a88780a9SRoi Dayan struct netlink_ext_ack *extack = f->common.extack; 2722a88780a9SRoi Dayan struct mlx5e_tc_flow_parse_attr *parse_attr; 2723a88780a9SRoi Dayan struct mlx5e_tc_flow *flow; 2724a88780a9SRoi Dayan int attr_size, err; 2725a88780a9SRoi Dayan 2726bf07aa73SPaul Blakey /* multi-chain not supported for NIC rules */ 2727bf07aa73SPaul Blakey if (!tc_cls_can_offload_and_chain0(priv->netdev, &f->common)) 2728bf07aa73SPaul Blakey return -EOPNOTSUPP; 2729bf07aa73SPaul Blakey 2730a88780a9SRoi Dayan flow_flags |= MLX5E_TC_FLOW_NIC; 2731a88780a9SRoi Dayan attr_size = sizeof(struct mlx5_nic_flow_attr); 2732a88780a9SRoi Dayan err = mlx5e_alloc_flow(priv, attr_size, f, flow_flags, 2733a88780a9SRoi Dayan &parse_attr, &flow); 2734a88780a9SRoi Dayan if (err) 2735a88780a9SRoi Dayan goto out; 2736a88780a9SRoi Dayan 2737d11afc26SOz Shlomo parse_attr->filter_dev = filter_dev; 273854c177caSOz Shlomo err = parse_cls_flower(flow->priv, flow, &parse_attr->spec, 273954c177caSOz Shlomo f, filter_dev); 2740d11afc26SOz Shlomo if (err) 2741d11afc26SOz Shlomo goto err_free; 2742d11afc26SOz Shlomo 2743a88780a9SRoi Dayan err = parse_tc_nic_actions(priv, f->exts, parse_attr, flow, extack); 2744a88780a9SRoi Dayan if (err) 2745a88780a9SRoi Dayan goto err_free; 2746a88780a9SRoi Dayan 2747a88780a9SRoi Dayan err = mlx5e_tc_add_nic_flow(priv, parse_attr, flow, extack); 2748a88780a9SRoi Dayan if (err) 2749a88780a9SRoi Dayan goto err_free; 2750a88780a9SRoi Dayan 2751a88780a9SRoi Dayan flow->flags |= MLX5E_TC_FLOW_OFFLOADED; 2752a88780a9SRoi Dayan kvfree(parse_attr); 2753a88780a9SRoi Dayan *__flow = flow; 2754a88780a9SRoi Dayan 2755a88780a9SRoi Dayan return 0; 2756a88780a9SRoi Dayan 2757a88780a9SRoi Dayan err_free: 2758a88780a9SRoi Dayan kfree(flow); 2759a88780a9SRoi Dayan kvfree(parse_attr); 2760a88780a9SRoi Dayan out: 2761a88780a9SRoi Dayan return err; 2762a88780a9SRoi Dayan } 2763a88780a9SRoi Dayan 2764a88780a9SRoi Dayan static int 2765a88780a9SRoi Dayan mlx5e_tc_add_flow(struct mlx5e_priv *priv, 2766a88780a9SRoi Dayan struct tc_cls_flower_offload *f, 2767a88780a9SRoi Dayan int flags, 2768d11afc26SOz Shlomo struct net_device *filter_dev, 2769a88780a9SRoi Dayan struct mlx5e_tc_flow **flow) 2770a88780a9SRoi Dayan { 2771a88780a9SRoi Dayan struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 27725dbe906fSPaul Blakey u16 flow_flags; 2773a88780a9SRoi Dayan int err; 2774a88780a9SRoi Dayan 2775a88780a9SRoi Dayan get_flags(flags, &flow_flags); 2776a88780a9SRoi Dayan 2777bf07aa73SPaul Blakey if (!tc_can_offload_extack(priv->netdev, f->common.extack)) 2778bf07aa73SPaul Blakey return -EOPNOTSUPP; 2779bf07aa73SPaul Blakey 2780a88780a9SRoi Dayan if (esw && esw->mode == SRIOV_OFFLOADS) 2781d11afc26SOz Shlomo err = mlx5e_add_fdb_flow(priv, f, flow_flags, 2782d11afc26SOz Shlomo filter_dev, flow); 2783a88780a9SRoi Dayan else 2784d11afc26SOz Shlomo err = mlx5e_add_nic_flow(priv, f, flow_flags, 2785d11afc26SOz Shlomo filter_dev, flow); 2786a88780a9SRoi Dayan 2787a88780a9SRoi Dayan return err; 2788a88780a9SRoi Dayan } 2789a88780a9SRoi Dayan 279071d82d2aSOz Shlomo int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv, 2791a88780a9SRoi Dayan struct tc_cls_flower_offload *f, int flags) 2792a88780a9SRoi Dayan { 2793a88780a9SRoi Dayan struct netlink_ext_ack *extack = f->common.extack; 2794a88780a9SRoi Dayan struct rhashtable *tc_ht = get_tc_ht(priv); 2795a88780a9SRoi Dayan struct mlx5e_tc_flow *flow; 2796a88780a9SRoi Dayan int err = 0; 2797a88780a9SRoi Dayan 2798a88780a9SRoi Dayan flow = rhashtable_lookup_fast(tc_ht, &f->cookie, tc_ht_params); 2799a88780a9SRoi Dayan if (flow) { 2800a88780a9SRoi Dayan NL_SET_ERR_MSG_MOD(extack, 2801a88780a9SRoi Dayan "flow cookie already exists, ignoring"); 2802a88780a9SRoi Dayan netdev_warn_once(priv->netdev, 2803a88780a9SRoi Dayan "flow cookie %lx already exists, ignoring\n", 2804a88780a9SRoi Dayan f->cookie); 2805a88780a9SRoi Dayan goto out; 2806a88780a9SRoi Dayan } 2807a88780a9SRoi Dayan 2808d11afc26SOz Shlomo err = mlx5e_tc_add_flow(priv, f, flags, dev, &flow); 2809a88780a9SRoi Dayan if (err) 2810a88780a9SRoi Dayan goto out; 2811a88780a9SRoi Dayan 2812a88780a9SRoi Dayan err = rhashtable_insert_fast(tc_ht, &flow->node, tc_ht_params); 2813a88780a9SRoi Dayan if (err) 2814a88780a9SRoi Dayan goto err_free; 2815a88780a9SRoi Dayan 2816a88780a9SRoi Dayan return 0; 2817a88780a9SRoi Dayan 2818a88780a9SRoi Dayan err_free: 2819a88780a9SRoi Dayan mlx5e_tc_del_flow(priv, flow); 2820a88780a9SRoi Dayan kfree(flow); 2821a88780a9SRoi Dayan out: 2822e3a2b7edSAmir Vadai return err; 2823e3a2b7edSAmir Vadai } 2824e3a2b7edSAmir Vadai 28258f8ae895SOr Gerlitz #define DIRECTION_MASK (MLX5E_TC_INGRESS | MLX5E_TC_EGRESS) 28268f8ae895SOr Gerlitz #define FLOW_DIRECTION_MASK (MLX5E_TC_FLOW_INGRESS | MLX5E_TC_FLOW_EGRESS) 28278f8ae895SOr Gerlitz 28288f8ae895SOr Gerlitz static bool same_flow_direction(struct mlx5e_tc_flow *flow, int flags) 28298f8ae895SOr Gerlitz { 28308f8ae895SOr Gerlitz if ((flow->flags & FLOW_DIRECTION_MASK) == (flags & DIRECTION_MASK)) 28318f8ae895SOr Gerlitz return true; 28328f8ae895SOr Gerlitz 28338f8ae895SOr Gerlitz return false; 28348f8ae895SOr Gerlitz } 28358f8ae895SOr Gerlitz 283671d82d2aSOz Shlomo int mlx5e_delete_flower(struct net_device *dev, struct mlx5e_priv *priv, 283760bd4af8SOr Gerlitz struct tc_cls_flower_offload *f, int flags) 2838e3a2b7edSAmir Vadai { 283905866c82SOr Gerlitz struct rhashtable *tc_ht = get_tc_ht(priv); 2840e3a2b7edSAmir Vadai struct mlx5e_tc_flow *flow; 2841e3a2b7edSAmir Vadai 284205866c82SOr Gerlitz flow = rhashtable_lookup_fast(tc_ht, &f->cookie, tc_ht_params); 28438f8ae895SOr Gerlitz if (!flow || !same_flow_direction(flow, flags)) 2844e3a2b7edSAmir Vadai return -EINVAL; 2845e3a2b7edSAmir Vadai 284605866c82SOr Gerlitz rhashtable_remove_fast(tc_ht, &flow->node, tc_ht_params); 2847e3a2b7edSAmir Vadai 2848961e8979SRoi Dayan mlx5e_tc_del_flow(priv, flow); 2849e3a2b7edSAmir Vadai 2850e3a2b7edSAmir Vadai kfree(flow); 2851e3a2b7edSAmir Vadai 2852e3a2b7edSAmir Vadai return 0; 2853e3a2b7edSAmir Vadai } 2854e3a2b7edSAmir Vadai 285571d82d2aSOz Shlomo int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, 285660bd4af8SOr Gerlitz struct tc_cls_flower_offload *f, int flags) 2857aad7e08dSAmir Vadai { 285805866c82SOr Gerlitz struct rhashtable *tc_ht = get_tc_ht(priv); 2859aad7e08dSAmir Vadai struct mlx5e_tc_flow *flow; 2860aad7e08dSAmir Vadai struct mlx5_fc *counter; 2861aad7e08dSAmir Vadai u64 bytes; 2862aad7e08dSAmir Vadai u64 packets; 2863aad7e08dSAmir Vadai u64 lastuse; 2864aad7e08dSAmir Vadai 286505866c82SOr Gerlitz flow = rhashtable_lookup_fast(tc_ht, &f->cookie, tc_ht_params); 28668f8ae895SOr Gerlitz if (!flow || !same_flow_direction(flow, flags)) 2867aad7e08dSAmir Vadai return -EINVAL; 2868aad7e08dSAmir Vadai 28690b67a38fSHadar Hen Zion if (!(flow->flags & MLX5E_TC_FLOW_OFFLOADED)) 28700b67a38fSHadar Hen Zion return 0; 28710b67a38fSHadar Hen Zion 2872b8aee822SMark Bloch counter = mlx5e_tc_get_counter(flow); 2873aad7e08dSAmir Vadai if (!counter) 2874aad7e08dSAmir Vadai return 0; 2875aad7e08dSAmir Vadai 2876aad7e08dSAmir Vadai mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse); 2877aad7e08dSAmir Vadai 2878d897a638SJakub Kicinski tcf_exts_stats_update(f->exts, bytes, packets, lastuse); 2879fed06ee8SOr Gerlitz 2880aad7e08dSAmir Vadai return 0; 2881aad7e08dSAmir Vadai } 2882aad7e08dSAmir Vadai 28834d8fcf21SAlaa Hleihel static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv, 28844d8fcf21SAlaa Hleihel struct mlx5e_priv *peer_priv) 28854d8fcf21SAlaa Hleihel { 28864d8fcf21SAlaa Hleihel struct mlx5_core_dev *peer_mdev = peer_priv->mdev; 28874d8fcf21SAlaa Hleihel struct mlx5e_hairpin_entry *hpe; 28884d8fcf21SAlaa Hleihel u16 peer_vhca_id; 28894d8fcf21SAlaa Hleihel int bkt; 28904d8fcf21SAlaa Hleihel 28914d8fcf21SAlaa Hleihel if (!same_hw_devs(priv, peer_priv)) 28924d8fcf21SAlaa Hleihel return; 28934d8fcf21SAlaa Hleihel 28944d8fcf21SAlaa Hleihel peer_vhca_id = MLX5_CAP_GEN(peer_mdev, vhca_id); 28954d8fcf21SAlaa Hleihel 28964d8fcf21SAlaa Hleihel hash_for_each(priv->fs.tc.hairpin_tbl, bkt, hpe, hairpin_hlist) { 28974d8fcf21SAlaa Hleihel if (hpe->peer_vhca_id == peer_vhca_id) 28984d8fcf21SAlaa Hleihel hpe->hp->pair->peer_gone = true; 28994d8fcf21SAlaa Hleihel } 29004d8fcf21SAlaa Hleihel } 29014d8fcf21SAlaa Hleihel 29024d8fcf21SAlaa Hleihel static int mlx5e_tc_netdev_event(struct notifier_block *this, 29034d8fcf21SAlaa Hleihel unsigned long event, void *ptr) 29044d8fcf21SAlaa Hleihel { 29054d8fcf21SAlaa Hleihel struct net_device *ndev = netdev_notifier_info_to_dev(ptr); 29064d8fcf21SAlaa Hleihel struct mlx5e_flow_steering *fs; 29074d8fcf21SAlaa Hleihel struct mlx5e_priv *peer_priv; 29084d8fcf21SAlaa Hleihel struct mlx5e_tc_table *tc; 29094d8fcf21SAlaa Hleihel struct mlx5e_priv *priv; 29104d8fcf21SAlaa Hleihel 29114d8fcf21SAlaa Hleihel if (ndev->netdev_ops != &mlx5e_netdev_ops || 29124d8fcf21SAlaa Hleihel event != NETDEV_UNREGISTER || 29134d8fcf21SAlaa Hleihel ndev->reg_state == NETREG_REGISTERED) 29144d8fcf21SAlaa Hleihel return NOTIFY_DONE; 29154d8fcf21SAlaa Hleihel 29164d8fcf21SAlaa Hleihel tc = container_of(this, struct mlx5e_tc_table, netdevice_nb); 29174d8fcf21SAlaa Hleihel fs = container_of(tc, struct mlx5e_flow_steering, tc); 29184d8fcf21SAlaa Hleihel priv = container_of(fs, struct mlx5e_priv, fs); 29194d8fcf21SAlaa Hleihel peer_priv = netdev_priv(ndev); 29204d8fcf21SAlaa Hleihel if (priv == peer_priv || 29214d8fcf21SAlaa Hleihel !(priv->netdev->features & NETIF_F_HW_TC)) 29224d8fcf21SAlaa Hleihel return NOTIFY_DONE; 29234d8fcf21SAlaa Hleihel 29244d8fcf21SAlaa Hleihel mlx5e_tc_hairpin_update_dead_peer(priv, peer_priv); 29254d8fcf21SAlaa Hleihel 29264d8fcf21SAlaa Hleihel return NOTIFY_DONE; 29274d8fcf21SAlaa Hleihel } 29284d8fcf21SAlaa Hleihel 2929655dc3d2SOr Gerlitz int mlx5e_tc_nic_init(struct mlx5e_priv *priv) 2930e8f887acSAmir Vadai { 2931acff797cSMaor Gottlieb struct mlx5e_tc_table *tc = &priv->fs.tc; 29324d8fcf21SAlaa Hleihel int err; 2933e8f887acSAmir Vadai 293411c9c548SOr Gerlitz hash_init(tc->mod_hdr_tbl); 29355c65c564SOr Gerlitz hash_init(tc->hairpin_tbl); 293611c9c548SOr Gerlitz 29374d8fcf21SAlaa Hleihel err = rhashtable_init(&tc->ht, &tc_ht_params); 29384d8fcf21SAlaa Hleihel if (err) 29394d8fcf21SAlaa Hleihel return err; 29404d8fcf21SAlaa Hleihel 29414d8fcf21SAlaa Hleihel tc->netdevice_nb.notifier_call = mlx5e_tc_netdev_event; 29424d8fcf21SAlaa Hleihel if (register_netdevice_notifier(&tc->netdevice_nb)) { 29434d8fcf21SAlaa Hleihel tc->netdevice_nb.notifier_call = NULL; 29444d8fcf21SAlaa Hleihel mlx5_core_warn(priv->mdev, "Failed to register netdev notifier\n"); 29454d8fcf21SAlaa Hleihel } 29464d8fcf21SAlaa Hleihel 29474d8fcf21SAlaa Hleihel return err; 2948e8f887acSAmir Vadai } 2949e8f887acSAmir Vadai 2950e8f887acSAmir Vadai static void _mlx5e_tc_del_flow(void *ptr, void *arg) 2951e8f887acSAmir Vadai { 2952e8f887acSAmir Vadai struct mlx5e_tc_flow *flow = ptr; 2953655dc3d2SOr Gerlitz struct mlx5e_priv *priv = flow->priv; 2954e8f887acSAmir Vadai 2955961e8979SRoi Dayan mlx5e_tc_del_flow(priv, flow); 2956e8f887acSAmir Vadai kfree(flow); 2957e8f887acSAmir Vadai } 2958e8f887acSAmir Vadai 2959655dc3d2SOr Gerlitz void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv) 2960e8f887acSAmir Vadai { 2961acff797cSMaor Gottlieb struct mlx5e_tc_table *tc = &priv->fs.tc; 2962e8f887acSAmir Vadai 29634d8fcf21SAlaa Hleihel if (tc->netdevice_nb.notifier_call) 29644d8fcf21SAlaa Hleihel unregister_netdevice_notifier(&tc->netdevice_nb); 29654d8fcf21SAlaa Hleihel 2966655dc3d2SOr Gerlitz rhashtable_free_and_destroy(&tc->ht, _mlx5e_tc_del_flow, NULL); 2967e8f887acSAmir Vadai 2968acff797cSMaor Gottlieb if (!IS_ERR_OR_NULL(tc->t)) { 2969acff797cSMaor Gottlieb mlx5_destroy_flow_table(tc->t); 2970acff797cSMaor Gottlieb tc->t = NULL; 2971e8f887acSAmir Vadai } 2972e8f887acSAmir Vadai } 2973655dc3d2SOr Gerlitz 2974655dc3d2SOr Gerlitz int mlx5e_tc_esw_init(struct rhashtable *tc_ht) 2975655dc3d2SOr Gerlitz { 2976655dc3d2SOr Gerlitz return rhashtable_init(tc_ht, &tc_ht_params); 2977655dc3d2SOr Gerlitz } 2978655dc3d2SOr Gerlitz 2979655dc3d2SOr Gerlitz void mlx5e_tc_esw_cleanup(struct rhashtable *tc_ht) 2980655dc3d2SOr Gerlitz { 2981655dc3d2SOr Gerlitz rhashtable_free_and_destroy(tc_ht, _mlx5e_tc_del_flow, NULL); 2982655dc3d2SOr Gerlitz } 298301252a27SOr Gerlitz 298401252a27SOr Gerlitz int mlx5e_tc_num_filters(struct mlx5e_priv *priv) 298501252a27SOr Gerlitz { 298601252a27SOr Gerlitz struct rhashtable *tc_ht = get_tc_ht(priv); 298701252a27SOr Gerlitz 298801252a27SOr Gerlitz return atomic_read(&tc_ht->nelems); 298901252a27SOr Gerlitz } 2990