1101f4de9SOz Shlomo /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2101f4de9SOz Shlomo /* Copyright (c) 2018 Mellanox Technologies. */
3101f4de9SOz Shlomo
448d67543SGuillaume Nault #include <net/inet_ecn.h>
5101f4de9SOz Shlomo #include <net/vxlan.h>
6df2ef3bfSOz Shlomo #include <net/gre.h>
79272e3dfSYevgeny Kliteynik #include <net/geneve.h>
8f828ca6aSEli Cohen #include <net/bareudp.h>
9101f4de9SOz Shlomo #include "en/tc_tun.h"
100d9f9647SVlad Buslov #include "en/tc_priv.h"
11f6dc1264SPaul Blakey #include "en_tc.h"
12768c3667SVlad Buslov #include "rep/tc.h"
13549c243eSVlad Buslov #include "rep/neigh.h"
14016c8946SJakub Kicinski #include "lag/lag.h"
15016c8946SJakub Kicinski #include "lag/mp.h"
16101f4de9SOz Shlomo
176717986eSVlad Buslov struct mlx5e_tc_tun_route_attr {
186717986eSVlad Buslov struct net_device *out_dev;
196717986eSVlad Buslov struct net_device *route_dev;
206717986eSVlad Buslov union {
216717986eSVlad Buslov struct flowi4 fl4;
226717986eSVlad Buslov struct flowi6 fl6;
236717986eSVlad Buslov } fl;
246717986eSVlad Buslov struct neighbour *n;
256717986eSVlad Buslov u8 ttl;
266717986eSVlad Buslov };
276717986eSVlad Buslov
286717986eSVlad Buslov #define TC_TUN_ROUTE_ATTR_INIT(name) struct mlx5e_tc_tun_route_attr name = {}
296717986eSVlad Buslov
mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr * attr)306717986eSVlad Buslov static void mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr *attr)
316717986eSVlad Buslov {
326717986eSVlad Buslov if (attr->n)
336717986eSVlad Buslov neigh_release(attr->n);
346717986eSVlad Buslov if (attr->route_dev)
356717986eSVlad Buslov dev_put(attr->route_dev);
366717986eSVlad Buslov }
376717986eSVlad Buslov
mlx5e_get_tc_tun(struct net_device * tunnel_dev)38d386939aSYevgeny Kliteynik struct mlx5e_tc_tunnel *mlx5e_get_tc_tun(struct net_device *tunnel_dev)
39d386939aSYevgeny Kliteynik {
40d386939aSYevgeny Kliteynik if (netif_is_vxlan(tunnel_dev))
41d386939aSYevgeny Kliteynik return &vxlan_tunnel;
429272e3dfSYevgeny Kliteynik else if (netif_is_geneve(tunnel_dev))
439272e3dfSYevgeny Kliteynik return &geneve_tunnel;
44d386939aSYevgeny Kliteynik else if (netif_is_gretap(tunnel_dev) ||
45d386939aSYevgeny Kliteynik netif_is_ip6gretap(tunnel_dev))
46d386939aSYevgeny Kliteynik return &gre_tunnel;
47f828ca6aSEli Cohen else if (netif_is_bareudp(tunnel_dev))
48f828ca6aSEli Cohen return &mplsoudp_tunnel;
49d386939aSYevgeny Kliteynik else
50d386939aSYevgeny Kliteynik return NULL;
51d386939aSYevgeny Kliteynik }
52d386939aSYevgeny Kliteynik
get_route_and_out_devs(struct mlx5e_priv * priv,struct net_device * dev,struct net_device ** route_dev,struct net_device ** out_dev)53442e1228SEli Britstein static int get_route_and_out_devs(struct mlx5e_priv *priv,
54442e1228SEli Britstein struct net_device *dev,
55442e1228SEli Britstein struct net_device **route_dev,
56442e1228SEli Britstein struct net_device **out_dev)
57442e1228SEli Britstein {
5845e7d4c0SEli Britstein struct net_device *uplink_dev, *uplink_upper, *real_dev;
59442e1228SEli Britstein struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
60442e1228SEli Britstein bool dst_is_lag_dev;
61442e1228SEli Britstein
6245e7d4c0SEli Britstein real_dev = is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : dev;
63442e1228SEli Britstein uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
64fa833bd5SVlad Buslov
65fa833bd5SVlad Buslov rcu_read_lock();
66fa833bd5SVlad Buslov uplink_upper = netdev_master_upper_dev_get_rcu(uplink_dev);
67fa833bd5SVlad Buslov /* mlx5_lag_is_sriov() is a blocking function which can't be called
68fa833bd5SVlad Buslov * while holding rcu read lock. Take the net_device for correctness
69fa833bd5SVlad Buslov * sake.
70fa833bd5SVlad Buslov */
71fa833bd5SVlad Buslov if (uplink_upper)
72fa833bd5SVlad Buslov dev_hold(uplink_upper);
73fa833bd5SVlad Buslov rcu_read_unlock();
74fa833bd5SVlad Buslov
75442e1228SEli Britstein dst_is_lag_dev = (uplink_upper &&
76442e1228SEli Britstein netif_is_lag_master(uplink_upper) &&
7745e7d4c0SEli Britstein real_dev == uplink_upper &&
78442e1228SEli Britstein mlx5_lag_is_sriov(priv->mdev));
79fa833bd5SVlad Buslov if (uplink_upper)
80fa833bd5SVlad Buslov dev_put(uplink_upper);
81442e1228SEli Britstein
82442e1228SEli Britstein /* if the egress device isn't on the same HW e-switch or
83442e1228SEli Britstein * it's a LAG device, use the uplink
84442e1228SEli Britstein */
85233fd212SEli Britstein *route_dev = dev;
8645e7d4c0SEli Britstein if (!netdev_port_same_parent_id(priv->netdev, real_dev) ||
87100ad4e2SAriel Levkovich dst_is_lag_dev || is_vlan_dev(*route_dev) ||
88100ad4e2SAriel Levkovich netif_is_ovs_master(*route_dev))
89442e1228SEli Britstein *out_dev = uplink_dev;
90f6dc1264SPaul Blakey else if (mlx5e_eswitch_rep(dev) &&
91f6dc1264SPaul Blakey mlx5e_is_valid_eswitch_fwd_dev(priv, dev))
92e32ee6c7SEli Britstein *out_dev = *route_dev;
93442e1228SEli Britstein else
94e32ee6c7SEli Britstein return -EOPNOTSUPP;
95442e1228SEli Britstein
96197c0002SRoi Dayan if (!mlx5e_eswitch_uplink_rep(*out_dev))
975e0060b1SDmytro Linkin return -EOPNOTSUPP;
985e0060b1SDmytro Linkin
998ce81fc0SRoi Dayan if (mlx5e_eswitch_uplink_rep(priv->netdev) && *out_dev != priv->netdev &&
1008ce81fc0SRoi Dayan !mlx5_lag_is_mpesw(priv->mdev))
10187b51810SEli Cohen return -EOPNOTSUPP;
10287b51810SEli Cohen
103442e1228SEli Britstein return 0;
104442e1228SEli Britstein }
105442e1228SEli Britstein
mlx5e_route_lookup_ipv4_get(struct mlx5e_priv * priv,struct net_device * dev,struct mlx5e_tc_tun_route_attr * attr)10678c906e4SVlad Buslov static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv,
107819c319cSChris Mi struct net_device *dev,
1086717986eSVlad Buslov struct mlx5e_tc_tun_route_attr *attr)
109101f4de9SOz Shlomo {
1106717986eSVlad Buslov struct net_device *route_dev;
1116717986eSVlad Buslov struct net_device *out_dev;
112e6014afdSEli Cohen struct neighbour *n;
113101f4de9SOz Shlomo struct rtable *rt;
114101f4de9SOz Shlomo
115101f4de9SOz Shlomo #if IS_ENABLED(CONFIG_INET)
1165fb091e8SRoi Dayan struct mlx5_core_dev *mdev = priv->mdev;
1175fb091e8SRoi Dayan struct net_device *uplink_dev;
118101f4de9SOz Shlomo int ret;
119101f4de9SOz Shlomo
1205fb091e8SRoi Dayan if (mlx5_lag_is_multipath(mdev)) {
1215fb091e8SRoi Dayan struct mlx5_eswitch *esw = mdev->priv.eswitch;
1225fb091e8SRoi Dayan
1235fb091e8SRoi Dayan uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
1246717986eSVlad Buslov attr->fl.fl4.flowi4_oif = uplink_dev->ifindex;
1252f8ec867SChris Mi } else {
126819c319cSChris Mi struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(dev);
1272f8ec867SChris Mi
1282f8ec867SChris Mi if (tunnel && tunnel->get_remote_ifindex)
129819c319cSChris Mi attr->fl.fl4.flowi4_oif = tunnel->get_remote_ifindex(dev);
1305fb091e8SRoi Dayan }
1315fb091e8SRoi Dayan
132819c319cSChris Mi rt = ip_route_output_key(dev_net(dev), &attr->fl.fl4);
1332639324aSTang Bin if (IS_ERR(rt))
1342639324aSTang Bin return PTR_ERR(rt);
1355fb091e8SRoi Dayan
136c623c95aSRoi Dayan if (rt->rt_type != RTN_UNICAST) {
137c623c95aSRoi Dayan ret = -ENETUNREACH;
138c623c95aSRoi Dayan goto err_rt_release;
139c623c95aSRoi Dayan }
140c623c95aSRoi Dayan
1412347cee8SParav Pandit if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) {
1426717986eSVlad Buslov ret = -ENETUNREACH;
1436717986eSVlad Buslov goto err_rt_release;
1442347cee8SParav Pandit }
145101f4de9SOz Shlomo #else
146101f4de9SOz Shlomo return -EOPNOTSUPP;
147101f4de9SOz Shlomo #endif
148491c37e4SRabie Loulou
1496717986eSVlad Buslov ret = get_route_and_out_devs(priv, rt->dst.dev, &route_dev, &out_dev);
1506717986eSVlad Buslov if (ret < 0)
1516717986eSVlad Buslov goto err_rt_release;
1526717986eSVlad Buslov dev_hold(route_dev);
1536717986eSVlad Buslov
1546717986eSVlad Buslov if (!attr->ttl)
1556717986eSVlad Buslov attr->ttl = ip4_dst_hoplimit(&rt->dst);
1566717986eSVlad Buslov n = dst_neigh_lookup(&rt->dst, &attr->fl.fl4.daddr);
1576717986eSVlad Buslov if (!n) {
1586717986eSVlad Buslov ret = -ENOMEM;
1596717986eSVlad Buslov goto err_dev_release;
1606717986eSVlad Buslov }
1616717986eSVlad Buslov
1626717986eSVlad Buslov ip_rt_put(rt);
1636717986eSVlad Buslov attr->route_dev = route_dev;
1646717986eSVlad Buslov attr->out_dev = out_dev;
1656717986eSVlad Buslov attr->n = n;
1666717986eSVlad Buslov return 0;
1676717986eSVlad Buslov
1686717986eSVlad Buslov err_dev_release:
1696717986eSVlad Buslov dev_put(route_dev);
1706717986eSVlad Buslov err_rt_release:
1712347cee8SParav Pandit ip_rt_put(rt);
172442e1228SEli Britstein return ret;
1732347cee8SParav Pandit }
174101f4de9SOz Shlomo
mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr * attr)1756717986eSVlad Buslov static void mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr *attr)
17678c906e4SVlad Buslov {
1776717986eSVlad Buslov mlx5e_tc_tun_route_attr_cleanup(attr);
17878c906e4SVlad Buslov }
17978c906e4SVlad Buslov
mlx5e_netdev_kind(struct net_device * dev)180101f4de9SOz Shlomo static const char *mlx5e_netdev_kind(struct net_device *dev)
181101f4de9SOz Shlomo {
182101f4de9SOz Shlomo if (dev->rtnl_link_ops)
183101f4de9SOz Shlomo return dev->rtnl_link_ops->kind;
184101f4de9SOz Shlomo else
1851b18b781STonghao Zhang return "unknown";
186101f4de9SOz Shlomo }
187101f4de9SOz Shlomo
mlx5e_gen_ip_tunnel_header(char buf[],__u8 * ip_proto,struct mlx5e_encap_entry * e)188101f4de9SOz Shlomo static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto,
189101f4de9SOz Shlomo struct mlx5e_encap_entry *e)
190101f4de9SOz Shlomo {
191d386939aSYevgeny Kliteynik if (!e->tunnel) {
192d386939aSYevgeny Kliteynik pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n");
193d386939aSYevgeny Kliteynik return -EOPNOTSUPP;
194101f4de9SOz Shlomo }
195101f4de9SOz Shlomo
196d386939aSYevgeny Kliteynik return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e);
197101f4de9SOz Shlomo }
198101f4de9SOz Shlomo
gen_eth_tnl_hdr(char * buf,struct net_device * dev,struct mlx5e_encap_entry * e,u16 proto)19905ada1adSEli Britstein static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev,
20005ada1adSEli Britstein struct mlx5e_encap_entry *e,
20105ada1adSEli Britstein u16 proto)
20205ada1adSEli Britstein {
20305ada1adSEli Britstein struct ethhdr *eth = (struct ethhdr *)buf;
204aa331450SEli Britstein char *ip;
20505ada1adSEli Britstein
20605ada1adSEli Britstein ether_addr_copy(eth->h_dest, e->h_dest);
20705ada1adSEli Britstein ether_addr_copy(eth->h_source, dev->dev_addr);
208aa331450SEli Britstein if (is_vlan_dev(dev)) {
209aa331450SEli Britstein struct vlan_hdr *vlan = (struct vlan_hdr *)
210aa331450SEli Britstein ((char *)eth + ETH_HLEN);
211aa331450SEli Britstein ip = (char *)vlan + VLAN_HLEN;
212aa331450SEli Britstein eth->h_proto = vlan_dev_vlan_proto(dev);
213aa331450SEli Britstein vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev));
214aa331450SEli Britstein vlan->h_vlan_encapsulated_proto = htons(proto);
215aa331450SEli Britstein } else {
21605ada1adSEli Britstein eth->h_proto = htons(proto);
217aa331450SEli Britstein ip = (char *)eth + ETH_HLEN;
218aa331450SEli Britstein }
21905ada1adSEli Britstein
220aa331450SEli Britstein return ip;
22105ada1adSEli Britstein }
22205ada1adSEli Britstein
mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)223101f4de9SOz Shlomo int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv,
224101f4de9SOz Shlomo struct net_device *mirred_dev,
225101f4de9SOz Shlomo struct mlx5e_encap_entry *e)
226101f4de9SOz Shlomo {
227101f4de9SOz Shlomo int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
2281f6da306SYevgeny Kliteynik const struct ip_tunnel_key *tun_key = &e->tun_info->key;
2293f3f05abSYevgeny Kliteynik struct mlx5_pkt_reformat_params reformat_params;
2302221d954SVlad Buslov struct mlx5e_neigh m_neigh = {};
2316717986eSVlad Buslov TC_TUN_ROUTE_ATTR_INIT(attr);
232c7bcb277SEli Britstein int ipv4_encap_size;
233101f4de9SOz Shlomo char *encap_header;
234101f4de9SOz Shlomo struct iphdr *ip;
2356717986eSVlad Buslov u8 nud_state;
236101f4de9SOz Shlomo int err;
237101f4de9SOz Shlomo
238101f4de9SOz Shlomo /* add the IP fields */
23948d67543SGuillaume Nault attr.fl.fl4.flowi4_tos = tun_key->tos & ~INET_ECN_MASK;
2406717986eSVlad Buslov attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
2416717986eSVlad Buslov attr.fl.fl4.saddr = tun_key->u.ipv4.src;
2426717986eSVlad Buslov attr.ttl = tun_key->ttl;
243101f4de9SOz Shlomo
2446717986eSVlad Buslov err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
245101f4de9SOz Shlomo if (err)
246c7bcb277SEli Britstein return err;
247c7bcb277SEli Britstein
248c7bcb277SEli Britstein ipv4_encap_size =
2496717986eSVlad Buslov (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
250c7bcb277SEli Britstein sizeof(struct iphdr) +
251d386939aSYevgeny Kliteynik e->tunnel->calc_hlen(e);
252c7bcb277SEli Britstein
253c7bcb277SEli Britstein if (max_encap_size < ipv4_encap_size) {
254c7bcb277SEli Britstein mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
255c7bcb277SEli Britstein ipv4_encap_size, max_encap_size);
25685bf490aSEli Cohen err = -EOPNOTSUPP;
25790ac2458SEli Cohen goto release_neigh;
258c7bcb277SEli Britstein }
259c7bcb277SEli Britstein
260c7bcb277SEli Britstein encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
26185bf490aSEli Cohen if (!encap_header) {
26285bf490aSEli Cohen err = -ENOMEM;
26390ac2458SEli Cohen goto release_neigh;
26485bf490aSEli Cohen }
265101f4de9SOz Shlomo
2662221d954SVlad Buslov m_neigh.family = attr.n->ops->family;
2672221d954SVlad Buslov memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
2686717986eSVlad Buslov e->out_dev = attr.out_dev;
2696717986eSVlad Buslov e->route_dev_ifindex = attr.route_dev->ifindex;
270101f4de9SOz Shlomo
271101f4de9SOz Shlomo /* It's important to add the neigh to the hash table before checking
272101f4de9SOz Shlomo * the neigh validity state. So if we'll get a notification, in case the
273101f4de9SOz Shlomo * neigh changes it's validity state, we would find the relevant neigh
274101f4de9SOz Shlomo * in the hash.
275101f4de9SOz Shlomo */
2762221d954SVlad Buslov err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
277101f4de9SOz Shlomo if (err)
278101f4de9SOz Shlomo goto free_encap;
279101f4de9SOz Shlomo
2806717986eSVlad Buslov read_lock_bh(&attr.n->lock);
2816717986eSVlad Buslov nud_state = attr.n->nud_state;
2826717986eSVlad Buslov ether_addr_copy(e->h_dest, attr.n->ha);
2836717986eSVlad Buslov read_unlock_bh(&attr.n->lock);
284101f4de9SOz Shlomo
285101f4de9SOz Shlomo /* add ethernet header */
2866717986eSVlad Buslov ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
28705ada1adSEli Britstein ETH_P_IP);
288101f4de9SOz Shlomo
289101f4de9SOz Shlomo /* add ip header */
290101f4de9SOz Shlomo ip->tos = tun_key->tos;
291101f4de9SOz Shlomo ip->version = 0x4;
292101f4de9SOz Shlomo ip->ihl = 0x5;
2936717986eSVlad Buslov ip->ttl = attr.ttl;
2946717986eSVlad Buslov ip->daddr = attr.fl.fl4.daddr;
2956717986eSVlad Buslov ip->saddr = attr.fl.fl4.saddr;
296101f4de9SOz Shlomo
297101f4de9SOz Shlomo /* add tunneling protocol header */
298101f4de9SOz Shlomo err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
299101f4de9SOz Shlomo &ip->protocol, e);
300101f4de9SOz Shlomo if (err)
301101f4de9SOz Shlomo goto destroy_neigh_entry;
302101f4de9SOz Shlomo
303*9c8d46bcSVlad Buslov e->encap_size = ipv4_encap_size;
304*9c8d46bcSVlad Buslov e->encap_header = encap_header;
305*9c8d46bcSVlad Buslov
306101f4de9SOz Shlomo if (!(nud_state & NUD_VALID)) {
3076717986eSVlad Buslov neigh_event_send(attr.n, NULL);
3080ad060eeSRoi Dayan /* the encap entry will be made valid on neigh update event
3090ad060eeSRoi Dayan * and not used before that.
3100ad060eeSRoi Dayan */
31190ac2458SEli Cohen goto release_neigh;
312101f4de9SOz Shlomo }
3133f3f05abSYevgeny Kliteynik
3143f3f05abSYevgeny Kliteynik memset(&reformat_params, 0, sizeof(reformat_params));
3153f3f05abSYevgeny Kliteynik reformat_params.type = e->reformat_type;
3163f3f05abSYevgeny Kliteynik reformat_params.size = ipv4_encap_size;
3173f3f05abSYevgeny Kliteynik reformat_params.data = encap_header;
3183f3f05abSYevgeny Kliteynik e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
3192b688ea5SMaor Gottlieb MLX5_FLOW_NAMESPACE_FDB);
3202b688ea5SMaor Gottlieb if (IS_ERR(e->pkt_reformat)) {
3212b688ea5SMaor Gottlieb err = PTR_ERR(e->pkt_reformat);
322101f4de9SOz Shlomo goto destroy_neigh_entry;
3232b688ea5SMaor Gottlieb }
324101f4de9SOz Shlomo
325101f4de9SOz Shlomo e->flags |= MLX5_ENCAP_ENTRY_VALID;
3266717986eSVlad Buslov mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
3276717986eSVlad Buslov mlx5e_route_lookup_ipv4_put(&attr);
328101f4de9SOz Shlomo return err;
329101f4de9SOz Shlomo
330101f4de9SOz Shlomo destroy_neigh_entry:
331101f4de9SOz Shlomo mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
332101f4de9SOz Shlomo free_encap:
333101f4de9SOz Shlomo kfree(encap_header);
33490ac2458SEli Cohen release_neigh:
3356717986eSVlad Buslov mlx5e_route_lookup_ipv4_put(&attr);
336101f4de9SOz Shlomo return err;
337101f4de9SOz Shlomo }
338101f4de9SOz Shlomo
mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)339c7b9038dSVlad Buslov int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv,
340c7b9038dSVlad Buslov struct net_device *mirred_dev,
341c7b9038dSVlad Buslov struct mlx5e_encap_entry *e)
342c7b9038dSVlad Buslov {
343c7b9038dSVlad Buslov int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
344c7b9038dSVlad Buslov const struct ip_tunnel_key *tun_key = &e->tun_info->key;
3453f3f05abSYevgeny Kliteynik struct mlx5_pkt_reformat_params reformat_params;
346c7b9038dSVlad Buslov TC_TUN_ROUTE_ATTR_INIT(attr);
347c7b9038dSVlad Buslov int ipv4_encap_size;
348c7b9038dSVlad Buslov char *encap_header;
349c7b9038dSVlad Buslov struct iphdr *ip;
350c7b9038dSVlad Buslov u8 nud_state;
351c7b9038dSVlad Buslov int err;
352c7b9038dSVlad Buslov
353c7b9038dSVlad Buslov /* add the IP fields */
35448d67543SGuillaume Nault attr.fl.fl4.flowi4_tos = tun_key->tos & ~INET_ECN_MASK;
355c7b9038dSVlad Buslov attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
356c7b9038dSVlad Buslov attr.fl.fl4.saddr = tun_key->u.ipv4.src;
357c7b9038dSVlad Buslov attr.ttl = tun_key->ttl;
358c7b9038dSVlad Buslov
359c7b9038dSVlad Buslov err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
360c7b9038dSVlad Buslov if (err)
361c7b9038dSVlad Buslov return err;
362c7b9038dSVlad Buslov
363c7b9038dSVlad Buslov ipv4_encap_size =
364c7b9038dSVlad Buslov (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
365c7b9038dSVlad Buslov sizeof(struct iphdr) +
366c7b9038dSVlad Buslov e->tunnel->calc_hlen(e);
367c7b9038dSVlad Buslov
368c7b9038dSVlad Buslov if (max_encap_size < ipv4_encap_size) {
369c7b9038dSVlad Buslov mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
370c7b9038dSVlad Buslov ipv4_encap_size, max_encap_size);
371c7b9038dSVlad Buslov err = -EOPNOTSUPP;
372c7b9038dSVlad Buslov goto release_neigh;
373c7b9038dSVlad Buslov }
374c7b9038dSVlad Buslov
375c7b9038dSVlad Buslov encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
376c7b9038dSVlad Buslov if (!encap_header) {
377c7b9038dSVlad Buslov err = -ENOMEM;
378c7b9038dSVlad Buslov goto release_neigh;
379c7b9038dSVlad Buslov }
380c7b9038dSVlad Buslov
381c7b9038dSVlad Buslov e->route_dev_ifindex = attr.route_dev->ifindex;
382c7b9038dSVlad Buslov
383c7b9038dSVlad Buslov read_lock_bh(&attr.n->lock);
384c7b9038dSVlad Buslov nud_state = attr.n->nud_state;
385c7b9038dSVlad Buslov ether_addr_copy(e->h_dest, attr.n->ha);
386c7b9038dSVlad Buslov WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
387c7b9038dSVlad Buslov read_unlock_bh(&attr.n->lock);
388c7b9038dSVlad Buslov
389c7b9038dSVlad Buslov /* add ethernet header */
390c7b9038dSVlad Buslov ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
391c7b9038dSVlad Buslov ETH_P_IP);
392c7b9038dSVlad Buslov
393c7b9038dSVlad Buslov /* add ip header */
394c7b9038dSVlad Buslov ip->tos = tun_key->tos;
395c7b9038dSVlad Buslov ip->version = 0x4;
396c7b9038dSVlad Buslov ip->ihl = 0x5;
397c7b9038dSVlad Buslov ip->ttl = attr.ttl;
398c7b9038dSVlad Buslov ip->daddr = attr.fl.fl4.daddr;
399c7b9038dSVlad Buslov ip->saddr = attr.fl.fl4.saddr;
400c7b9038dSVlad Buslov
401c7b9038dSVlad Buslov /* add tunneling protocol header */
402c7b9038dSVlad Buslov err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
403c7b9038dSVlad Buslov &ip->protocol, e);
404c7b9038dSVlad Buslov if (err)
405c7b9038dSVlad Buslov goto free_encap;
406c7b9038dSVlad Buslov
407c47e9c56SVlad Buslov e->encap_size = ipv4_encap_size;
408c47e9c56SVlad Buslov kfree(e->encap_header);
409c47e9c56SVlad Buslov e->encap_header = encap_header;
410c47e9c56SVlad Buslov
411c7b9038dSVlad Buslov if (!(nud_state & NUD_VALID)) {
412c7b9038dSVlad Buslov neigh_event_send(attr.n, NULL);
413c7b9038dSVlad Buslov /* the encap entry will be made valid on neigh update event
414c7b9038dSVlad Buslov * and not used before that.
415c7b9038dSVlad Buslov */
416c47e9c56SVlad Buslov goto release_neigh;
417c7b9038dSVlad Buslov }
4183f3f05abSYevgeny Kliteynik
4193f3f05abSYevgeny Kliteynik memset(&reformat_params, 0, sizeof(reformat_params));
4203f3f05abSYevgeny Kliteynik reformat_params.type = e->reformat_type;
4213f3f05abSYevgeny Kliteynik reformat_params.size = ipv4_encap_size;
4223f3f05abSYevgeny Kliteynik reformat_params.data = encap_header;
4233f3f05abSYevgeny Kliteynik e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
424c7b9038dSVlad Buslov MLX5_FLOW_NAMESPACE_FDB);
425c7b9038dSVlad Buslov if (IS_ERR(e->pkt_reformat)) {
426c7b9038dSVlad Buslov err = PTR_ERR(e->pkt_reformat);
427c7b9038dSVlad Buslov goto free_encap;
428c7b9038dSVlad Buslov }
429c7b9038dSVlad Buslov
430c7b9038dSVlad Buslov e->flags |= MLX5_ENCAP_ENTRY_VALID;
431c7b9038dSVlad Buslov mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
432c7b9038dSVlad Buslov mlx5e_route_lookup_ipv4_put(&attr);
433c7b9038dSVlad Buslov return err;
434c7b9038dSVlad Buslov
435c7b9038dSVlad Buslov free_encap:
436c7b9038dSVlad Buslov kfree(encap_header);
437c7b9038dSVlad Buslov release_neigh:
438c7b9038dSVlad Buslov mlx5e_route_lookup_ipv4_put(&attr);
439c7b9038dSVlad Buslov return err;
440c7b9038dSVlad Buslov }
441c7b9038dSVlad Buslov
4425f9fc332SYueHaibing #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
mlx5e_route_lookup_ipv6_get(struct mlx5e_priv * priv,struct net_device * dev,struct mlx5e_tc_tun_route_attr * attr)44378c906e4SVlad Buslov static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv,
444819c319cSChris Mi struct net_device *dev,
4456717986eSVlad Buslov struct mlx5e_tc_tun_route_attr *attr)
4465f9fc332SYueHaibing {
447819c319cSChris Mi struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(dev);
4486717986eSVlad Buslov struct net_device *route_dev;
4496717986eSVlad Buslov struct net_device *out_dev;
4505f9fc332SYueHaibing struct dst_entry *dst;
4515f9fc332SYueHaibing struct neighbour *n;
4525f9fc332SYueHaibing int ret;
4535f9fc332SYueHaibing
4542f8ec867SChris Mi if (tunnel && tunnel->get_remote_ifindex)
455819c319cSChris Mi attr->fl.fl6.flowi6_oif = tunnel->get_remote_ifindex(dev);
456819c319cSChris Mi dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(dev), NULL, &attr->fl.fl6,
4576c8991f4SSabrina Dubroca NULL);
4586c8991f4SSabrina Dubroca if (IS_ERR(dst))
4596c8991f4SSabrina Dubroca return PTR_ERR(dst);
4605f9fc332SYueHaibing
4616717986eSVlad Buslov if (!attr->ttl)
4626717986eSVlad Buslov attr->ttl = ip6_dst_hoplimit(dst);
4635f9fc332SYueHaibing
4646717986eSVlad Buslov ret = get_route_and_out_devs(priv, dst->dev, &route_dev, &out_dev);
4656717986eSVlad Buslov if (ret < 0)
4666717986eSVlad Buslov goto err_dst_release;
4676717986eSVlad Buslov
4686717986eSVlad Buslov dev_hold(route_dev);
4696717986eSVlad Buslov n = dst_neigh_lookup(dst, &attr->fl.fl6.daddr);
4706717986eSVlad Buslov if (!n) {
4716717986eSVlad Buslov ret = -ENOMEM;
4726717986eSVlad Buslov goto err_dev_release;
4736717986eSVlad Buslov }
4746717986eSVlad Buslov
4756717986eSVlad Buslov dst_release(dst);
4766717986eSVlad Buslov attr->out_dev = out_dev;
4776717986eSVlad Buslov attr->route_dev = route_dev;
4786717986eSVlad Buslov attr->n = n;
4796717986eSVlad Buslov return 0;
4806717986eSVlad Buslov
4816717986eSVlad Buslov err_dev_release:
4826717986eSVlad Buslov dev_put(route_dev);
4836717986eSVlad Buslov err_dst_release:
4845f9fc332SYueHaibing dst_release(dst);
4855f9fc332SYueHaibing return ret;
4865f9fc332SYueHaibing }
4875f9fc332SYueHaibing
mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr * attr)4886717986eSVlad Buslov static void mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr *attr)
48978c906e4SVlad Buslov {
4906717986eSVlad Buslov mlx5e_tc_tun_route_attr_cleanup(attr);
49178c906e4SVlad Buslov }
49278c906e4SVlad Buslov
mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)493101f4de9SOz Shlomo int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv,
494101f4de9SOz Shlomo struct net_device *mirred_dev,
495101f4de9SOz Shlomo struct mlx5e_encap_entry *e)
496101f4de9SOz Shlomo {
497101f4de9SOz Shlomo int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
4981f6da306SYevgeny Kliteynik const struct ip_tunnel_key *tun_key = &e->tun_info->key;
4993f3f05abSYevgeny Kliteynik struct mlx5_pkt_reformat_params reformat_params;
5002221d954SVlad Buslov struct mlx5e_neigh m_neigh = {};
5016717986eSVlad Buslov TC_TUN_ROUTE_ATTR_INIT(attr);
502101f4de9SOz Shlomo struct ipv6hdr *ip6h;
503c7bcb277SEli Britstein int ipv6_encap_size;
504101f4de9SOz Shlomo char *encap_header;
5056717986eSVlad Buslov u8 nud_state;
506101f4de9SOz Shlomo int err;
507101f4de9SOz Shlomo
5086717986eSVlad Buslov attr.ttl = tun_key->ttl;
509bcb0da7fSMatthias May attr.fl.fl6.flowlabel = ip6_make_flowinfo(tun_key->tos, tun_key->label);
5106717986eSVlad Buslov attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
5116717986eSVlad Buslov attr.fl.fl6.saddr = tun_key->u.ipv6.src;
512101f4de9SOz Shlomo
5136717986eSVlad Buslov err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
514101f4de9SOz Shlomo if (err)
515c7bcb277SEli Britstein return err;
516c7bcb277SEli Britstein
517c7bcb277SEli Britstein ipv6_encap_size =
5186717986eSVlad Buslov (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
519c7bcb277SEli Britstein sizeof(struct ipv6hdr) +
520d386939aSYevgeny Kliteynik e->tunnel->calc_hlen(e);
521c7bcb277SEli Britstein
522c7bcb277SEli Britstein if (max_encap_size < ipv6_encap_size) {
523c7bcb277SEli Britstein mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
524c7bcb277SEli Britstein ipv6_encap_size, max_encap_size);
52585bf490aSEli Cohen err = -EOPNOTSUPP;
52690ac2458SEli Cohen goto release_neigh;
527c7bcb277SEli Britstein }
528c7bcb277SEli Britstein
529c7bcb277SEli Britstein encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
53085bf490aSEli Cohen if (!encap_header) {
53185bf490aSEli Cohen err = -ENOMEM;
53290ac2458SEli Cohen goto release_neigh;
53385bf490aSEli Cohen }
534101f4de9SOz Shlomo
5352221d954SVlad Buslov m_neigh.family = attr.n->ops->family;
5362221d954SVlad Buslov memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
5376717986eSVlad Buslov e->out_dev = attr.out_dev;
5386717986eSVlad Buslov e->route_dev_ifindex = attr.route_dev->ifindex;
539101f4de9SOz Shlomo
54039c538d6SCai Huoqing /* It's important to add the neigh to the hash table before checking
541101f4de9SOz Shlomo * the neigh validity state. So if we'll get a notification, in case the
542101f4de9SOz Shlomo * neigh changes it's validity state, we would find the relevant neigh
543101f4de9SOz Shlomo * in the hash.
544101f4de9SOz Shlomo */
5452221d954SVlad Buslov err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
546101f4de9SOz Shlomo if (err)
547101f4de9SOz Shlomo goto free_encap;
548101f4de9SOz Shlomo
5496717986eSVlad Buslov read_lock_bh(&attr.n->lock);
5506717986eSVlad Buslov nud_state = attr.n->nud_state;
5516717986eSVlad Buslov ether_addr_copy(e->h_dest, attr.n->ha);
5526717986eSVlad Buslov read_unlock_bh(&attr.n->lock);
553101f4de9SOz Shlomo
554101f4de9SOz Shlomo /* add ethernet header */
5556717986eSVlad Buslov ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
55605ada1adSEli Britstein ETH_P_IPV6);
557101f4de9SOz Shlomo
558101f4de9SOz Shlomo /* add ip header */
559101f4de9SOz Shlomo ip6_flow_hdr(ip6h, tun_key->tos, 0);
560101f4de9SOz Shlomo /* the HW fills up ipv6 payload len */
5616717986eSVlad Buslov ip6h->hop_limit = attr.ttl;
5626717986eSVlad Buslov ip6h->daddr = attr.fl.fl6.daddr;
5636717986eSVlad Buslov ip6h->saddr = attr.fl.fl6.saddr;
564101f4de9SOz Shlomo
565101f4de9SOz Shlomo /* add tunneling protocol header */
566101f4de9SOz Shlomo err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
567101f4de9SOz Shlomo &ip6h->nexthdr, e);
568101f4de9SOz Shlomo if (err)
569101f4de9SOz Shlomo goto destroy_neigh_entry;
570101f4de9SOz Shlomo
571*9c8d46bcSVlad Buslov e->encap_size = ipv6_encap_size;
572*9c8d46bcSVlad Buslov e->encap_header = encap_header;
573*9c8d46bcSVlad Buslov
574101f4de9SOz Shlomo if (!(nud_state & NUD_VALID)) {
5756717986eSVlad Buslov neigh_event_send(attr.n, NULL);
5760ad060eeSRoi Dayan /* the encap entry will be made valid on neigh update event
5770ad060eeSRoi Dayan * and not used before that.
5780ad060eeSRoi Dayan */
57990ac2458SEli Cohen goto release_neigh;
580101f4de9SOz Shlomo }
581101f4de9SOz Shlomo
5823f3f05abSYevgeny Kliteynik memset(&reformat_params, 0, sizeof(reformat_params));
5833f3f05abSYevgeny Kliteynik reformat_params.type = e->reformat_type;
5843f3f05abSYevgeny Kliteynik reformat_params.size = ipv6_encap_size;
5853f3f05abSYevgeny Kliteynik reformat_params.data = encap_header;
5863f3f05abSYevgeny Kliteynik e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
5872b688ea5SMaor Gottlieb MLX5_FLOW_NAMESPACE_FDB);
5882b688ea5SMaor Gottlieb if (IS_ERR(e->pkt_reformat)) {
5892b688ea5SMaor Gottlieb err = PTR_ERR(e->pkt_reformat);
590101f4de9SOz Shlomo goto destroy_neigh_entry;
5912b688ea5SMaor Gottlieb }
592101f4de9SOz Shlomo
593101f4de9SOz Shlomo e->flags |= MLX5_ENCAP_ENTRY_VALID;
5946717986eSVlad Buslov mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
5956717986eSVlad Buslov mlx5e_route_lookup_ipv6_put(&attr);
596101f4de9SOz Shlomo return err;
597101f4de9SOz Shlomo
598101f4de9SOz Shlomo destroy_neigh_entry:
599101f4de9SOz Shlomo mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
600101f4de9SOz Shlomo free_encap:
601101f4de9SOz Shlomo kfree(encap_header);
60290ac2458SEli Cohen release_neigh:
6036717986eSVlad Buslov mlx5e_route_lookup_ipv6_put(&attr);
604101f4de9SOz Shlomo return err;
605101f4de9SOz Shlomo }
606c7b9038dSVlad Buslov
mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)607c7b9038dSVlad Buslov int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv,
608c7b9038dSVlad Buslov struct net_device *mirred_dev,
609c7b9038dSVlad Buslov struct mlx5e_encap_entry *e)
610c7b9038dSVlad Buslov {
611c7b9038dSVlad Buslov int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
612c7b9038dSVlad Buslov const struct ip_tunnel_key *tun_key = &e->tun_info->key;
6133f3f05abSYevgeny Kliteynik struct mlx5_pkt_reformat_params reformat_params;
614c7b9038dSVlad Buslov TC_TUN_ROUTE_ATTR_INIT(attr);
615c7b9038dSVlad Buslov struct ipv6hdr *ip6h;
616c7b9038dSVlad Buslov int ipv6_encap_size;
617c7b9038dSVlad Buslov char *encap_header;
618c7b9038dSVlad Buslov u8 nud_state;
619c7b9038dSVlad Buslov int err;
620c7b9038dSVlad Buslov
621c7b9038dSVlad Buslov attr.ttl = tun_key->ttl;
622c7b9038dSVlad Buslov
623bcb0da7fSMatthias May attr.fl.fl6.flowlabel = ip6_make_flowinfo(tun_key->tos, tun_key->label);
624c7b9038dSVlad Buslov attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
625c7b9038dSVlad Buslov attr.fl.fl6.saddr = tun_key->u.ipv6.src;
626c7b9038dSVlad Buslov
627c7b9038dSVlad Buslov err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
628c7b9038dSVlad Buslov if (err)
629c7b9038dSVlad Buslov return err;
630c7b9038dSVlad Buslov
631c7b9038dSVlad Buslov ipv6_encap_size =
632c7b9038dSVlad Buslov (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
633c7b9038dSVlad Buslov sizeof(struct ipv6hdr) +
634c7b9038dSVlad Buslov e->tunnel->calc_hlen(e);
635c7b9038dSVlad Buslov
636c7b9038dSVlad Buslov if (max_encap_size < ipv6_encap_size) {
637c7b9038dSVlad Buslov mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
638c7b9038dSVlad Buslov ipv6_encap_size, max_encap_size);
639c7b9038dSVlad Buslov err = -EOPNOTSUPP;
640c7b9038dSVlad Buslov goto release_neigh;
641c7b9038dSVlad Buslov }
642c7b9038dSVlad Buslov
643c7b9038dSVlad Buslov encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
644c7b9038dSVlad Buslov if (!encap_header) {
645c7b9038dSVlad Buslov err = -ENOMEM;
646c7b9038dSVlad Buslov goto release_neigh;
647c7b9038dSVlad Buslov }
648c7b9038dSVlad Buslov
649c7b9038dSVlad Buslov e->route_dev_ifindex = attr.route_dev->ifindex;
650c7b9038dSVlad Buslov
651c7b9038dSVlad Buslov read_lock_bh(&attr.n->lock);
652c7b9038dSVlad Buslov nud_state = attr.n->nud_state;
653c7b9038dSVlad Buslov ether_addr_copy(e->h_dest, attr.n->ha);
654c7b9038dSVlad Buslov WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
655c7b9038dSVlad Buslov read_unlock_bh(&attr.n->lock);
656c7b9038dSVlad Buslov
657c7b9038dSVlad Buslov /* add ethernet header */
658c7b9038dSVlad Buslov ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
659c7b9038dSVlad Buslov ETH_P_IPV6);
660c7b9038dSVlad Buslov
661c7b9038dSVlad Buslov /* add ip header */
662c7b9038dSVlad Buslov ip6_flow_hdr(ip6h, tun_key->tos, 0);
663c7b9038dSVlad Buslov /* the HW fills up ipv6 payload len */
664c7b9038dSVlad Buslov ip6h->hop_limit = attr.ttl;
665c7b9038dSVlad Buslov ip6h->daddr = attr.fl.fl6.daddr;
666c7b9038dSVlad Buslov ip6h->saddr = attr.fl.fl6.saddr;
667c7b9038dSVlad Buslov
668c7b9038dSVlad Buslov /* add tunneling protocol header */
669c7b9038dSVlad Buslov err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
670c7b9038dSVlad Buslov &ip6h->nexthdr, e);
671c7b9038dSVlad Buslov if (err)
672c7b9038dSVlad Buslov goto free_encap;
673c7b9038dSVlad Buslov
674c47e9c56SVlad Buslov e->encap_size = ipv6_encap_size;
675c47e9c56SVlad Buslov kfree(e->encap_header);
676c47e9c56SVlad Buslov e->encap_header = encap_header;
677c47e9c56SVlad Buslov
678c7b9038dSVlad Buslov if (!(nud_state & NUD_VALID)) {
679c7b9038dSVlad Buslov neigh_event_send(attr.n, NULL);
680c7b9038dSVlad Buslov /* the encap entry will be made valid on neigh update event
681c7b9038dSVlad Buslov * and not used before that.
682c7b9038dSVlad Buslov */
683c47e9c56SVlad Buslov goto release_neigh;
684c7b9038dSVlad Buslov }
685c7b9038dSVlad Buslov
6863f3f05abSYevgeny Kliteynik memset(&reformat_params, 0, sizeof(reformat_params));
6873f3f05abSYevgeny Kliteynik reformat_params.type = e->reformat_type;
6883f3f05abSYevgeny Kliteynik reformat_params.size = ipv6_encap_size;
6893f3f05abSYevgeny Kliteynik reformat_params.data = encap_header;
6903f3f05abSYevgeny Kliteynik e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
691c7b9038dSVlad Buslov MLX5_FLOW_NAMESPACE_FDB);
692c7b9038dSVlad Buslov if (IS_ERR(e->pkt_reformat)) {
693c7b9038dSVlad Buslov err = PTR_ERR(e->pkt_reformat);
694c7b9038dSVlad Buslov goto free_encap;
695c7b9038dSVlad Buslov }
696c7b9038dSVlad Buslov
697c7b9038dSVlad Buslov e->flags |= MLX5_ENCAP_ENTRY_VALID;
698c7b9038dSVlad Buslov mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
699c7b9038dSVlad Buslov mlx5e_route_lookup_ipv6_put(&attr);
700c7b9038dSVlad Buslov return err;
701c7b9038dSVlad Buslov
702c7b9038dSVlad Buslov free_encap:
703c7b9038dSVlad Buslov kfree(encap_header);
704c7b9038dSVlad Buslov release_neigh:
705c7b9038dSVlad Buslov mlx5e_route_lookup_ipv6_put(&attr);
706c7b9038dSVlad Buslov return err;
707c7b9038dSVlad Buslov }
7085f9fc332SYueHaibing #endif
709101f4de9SOz Shlomo
mlx5e_tc_tun_route_lookup(struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * flow_attr,struct net_device * filter_dev)710a508728aSVlad Buslov int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv,
711a508728aSVlad Buslov struct mlx5_flow_spec *spec,
712819c319cSChris Mi struct mlx5_flow_attr *flow_attr,
713819c319cSChris Mi struct net_device *filter_dev)
714a508728aSVlad Buslov {
715a508728aSVlad Buslov struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr;
716e3fdc71bSAriel Levkovich struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
717b16eb3c8SAriel Levkovich struct mlx5e_tc_int_port *int_port;
718a508728aSVlad Buslov TC_TUN_ROUTE_ATTR_INIT(attr);
719a508728aSVlad Buslov u16 vport_num;
720a508728aSVlad Buslov int err = 0;
721a508728aSVlad Buslov
7221e74152eSRoi Dayan if (flow_attr->tun_ip_version == 4) {
723a508728aSVlad Buslov /* Addresses are swapped for decap */
724a508728aSVlad Buslov attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4;
725a508728aSVlad Buslov attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4;
726819c319cSChris Mi err = mlx5e_route_lookup_ipv4_get(priv, filter_dev, &attr);
727a508728aSVlad Buslov }
728a508728aSVlad Buslov #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
7291e74152eSRoi Dayan else if (flow_attr->tun_ip_version == 6) {
730a508728aSVlad Buslov /* Addresses are swapped for decap */
731a508728aSVlad Buslov attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6;
732a508728aSVlad Buslov attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6;
733819c319cSChris Mi err = mlx5e_route_lookup_ipv6_get(priv, filter_dev, &attr);
734a508728aSVlad Buslov }
735a508728aSVlad Buslov #endif
736a508728aSVlad Buslov else
737a508728aSVlad Buslov return 0;
738a508728aSVlad Buslov
739a508728aSVlad Buslov if (err)
740a508728aSVlad Buslov return err;
741a508728aSVlad Buslov
742b16eb3c8SAriel Levkovich if (attr.route_dev->netdev_ops == &mlx5e_netdev_ops &&
743b16eb3c8SAriel Levkovich mlx5e_tc_is_vf_tunnel(attr.out_dev, attr.route_dev)) {
744a508728aSVlad Buslov err = mlx5e_tc_query_route_vport(attr.out_dev, attr.route_dev, &vport_num);
745a508728aSVlad Buslov if (err)
746a508728aSVlad Buslov goto out;
747a508728aSVlad Buslov
748a508728aSVlad Buslov esw_attr->rx_tun_attr->decap_vport = vport_num;
749e3fdc71bSAriel Levkovich } else if (netif_is_ovs_master(attr.route_dev) && mlx5e_tc_int_port_supported(esw)) {
750b16eb3c8SAriel Levkovich int_port = mlx5e_tc_int_port_get(mlx5e_get_int_port_priv(priv),
751b16eb3c8SAriel Levkovich attr.route_dev->ifindex,
752b16eb3c8SAriel Levkovich MLX5E_TC_INT_PORT_INGRESS);
753b16eb3c8SAriel Levkovich if (IS_ERR(int_port)) {
754b16eb3c8SAriel Levkovich err = PTR_ERR(int_port);
755b16eb3c8SAriel Levkovich goto out;
756b16eb3c8SAriel Levkovich }
757b16eb3c8SAriel Levkovich esw_attr->int_port = int_port;
758b16eb3c8SAriel Levkovich }
759a508728aSVlad Buslov
760a508728aSVlad Buslov out:
7611e74152eSRoi Dayan if (flow_attr->tun_ip_version == 4)
762a508728aSVlad Buslov mlx5e_route_lookup_ipv4_put(&attr);
763a508728aSVlad Buslov #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
7641e74152eSRoi Dayan else if (flow_attr->tun_ip_version == 6)
765a508728aSVlad Buslov mlx5e_route_lookup_ipv6_put(&attr);
766a508728aSVlad Buslov #endif
767a508728aSVlad Buslov return err;
768a508728aSVlad Buslov }
769a508728aSVlad Buslov
mlx5e_tc_tun_device_to_offload(struct mlx5e_priv * priv,struct net_device * netdev)770101f4de9SOz Shlomo bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv,
771101f4de9SOz Shlomo struct net_device *netdev)
772101f4de9SOz Shlomo {
773d386939aSYevgeny Kliteynik struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev);
774101f4de9SOz Shlomo
775d386939aSYevgeny Kliteynik if (tunnel && tunnel->can_offload(priv))
776df2ef3bfSOz Shlomo return true;
777101f4de9SOz Shlomo else
778101f4de9SOz Shlomo return false;
779101f4de9SOz Shlomo }
780101f4de9SOz Shlomo
mlx5e_tc_tun_init_encap_attr(struct net_device * tunnel_dev,struct mlx5e_priv * priv,struct mlx5e_encap_entry * e,struct netlink_ext_ack * extack)781101f4de9SOz Shlomo int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev,
782101f4de9SOz Shlomo struct mlx5e_priv *priv,
783101f4de9SOz Shlomo struct mlx5e_encap_entry *e,
784101f4de9SOz Shlomo struct netlink_ext_ack *extack)
785101f4de9SOz Shlomo {
786d386939aSYevgeny Kliteynik struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev);
787101f4de9SOz Shlomo
788d386939aSYevgeny Kliteynik if (!tunnel) {
789101f4de9SOz Shlomo e->reformat_type = -1;
790101f4de9SOz Shlomo return -EOPNOTSUPP;
791101f4de9SOz Shlomo }
792101f4de9SOz Shlomo
793d386939aSYevgeny Kliteynik return tunnel->init_encap_attr(tunnel_dev, priv, e, extack);
794df2ef3bfSOz Shlomo }
795df2ef3bfSOz Shlomo
mlx5e_tc_tun_parse(struct net_device * filter_dev,struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,u8 * match_level)796101f4de9SOz Shlomo int mlx5e_tc_tun_parse(struct net_device *filter_dev,
797101f4de9SOz Shlomo struct mlx5e_priv *priv,
798101f4de9SOz Shlomo struct mlx5_flow_spec *spec,
799f9e30088SPablo Neira Ayuso struct flow_cls_offload *f,
800ea4cd837SPaul Blakey u8 *match_level)
801101f4de9SOz Shlomo {
802d386939aSYevgeny Kliteynik struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(filter_dev);
803ea4cd837SPaul Blakey struct flow_rule *rule = flow_cls_offload_flow_rule(f);
804ea4cd837SPaul Blakey void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
805ea4cd837SPaul Blakey outer_headers);
806ea4cd837SPaul Blakey void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
807ea4cd837SPaul Blakey outer_headers);
808ea4cd837SPaul Blakey struct netlink_ext_ack *extack = f->common.extack;
809101f4de9SOz Shlomo int err = 0;
810101f4de9SOz Shlomo
811d386939aSYevgeny Kliteynik if (!tunnel) {
812101f4de9SOz Shlomo netdev_warn(priv->netdev,
813d386939aSYevgeny Kliteynik "decapsulation offload is not supported for %s net device\n",
8141b18b781STonghao Zhang mlx5e_netdev_kind(filter_dev));
815d386939aSYevgeny Kliteynik err = -EOPNOTSUPP;
816d386939aSYevgeny Kliteynik goto out;
817d386939aSYevgeny Kliteynik }
8181b18b781STonghao Zhang
819d386939aSYevgeny Kliteynik *match_level = tunnel->match_level;
820d386939aSYevgeny Kliteynik
821d386939aSYevgeny Kliteynik if (tunnel->parse_udp_ports) {
822d386939aSYevgeny Kliteynik err = tunnel->parse_udp_ports(priv, spec, f,
823d386939aSYevgeny Kliteynik headers_c, headers_v);
824d386939aSYevgeny Kliteynik if (err)
825d386939aSYevgeny Kliteynik goto out;
826d386939aSYevgeny Kliteynik }
827d386939aSYevgeny Kliteynik
828d386939aSYevgeny Kliteynik if (tunnel->parse_tunnel) {
829d386939aSYevgeny Kliteynik err = tunnel->parse_tunnel(priv, spec, f,
830d386939aSYevgeny Kliteynik headers_c, headers_v);
831d386939aSYevgeny Kliteynik if (err)
832d386939aSYevgeny Kliteynik goto out;
833d386939aSYevgeny Kliteynik }
834d386939aSYevgeny Kliteynik
835ea4cd837SPaul Blakey if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
8364a5d5d73SEli Britstein struct flow_dissector_key_basic key_basic = {};
8374a5d5d73SEli Britstein struct flow_dissector_key_basic mask_basic = {
8384a5d5d73SEli Britstein .n_proto = htons(0xFFFF),
8394a5d5d73SEli Britstein };
8404a5d5d73SEli Britstein struct flow_match_basic match_basic = {
8414a5d5d73SEli Britstein .key = &key_basic, .mask = &mask_basic,
8424a5d5d73SEli Britstein };
843ea4cd837SPaul Blakey struct flow_match_control match;
844ea4cd837SPaul Blakey u16 addr_type;
845ea4cd837SPaul Blakey
846ea4cd837SPaul Blakey flow_rule_match_enc_control(rule, &match);
847ea4cd837SPaul Blakey addr_type = match.key->addr_type;
848ea4cd837SPaul Blakey
849ea4cd837SPaul Blakey /* For tunnel addr_type used same key id`s as for non-tunnel */
850ea4cd837SPaul Blakey if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
851ea4cd837SPaul Blakey struct flow_match_ipv4_addrs match;
852ea4cd837SPaul Blakey
853ea4cd837SPaul Blakey flow_rule_match_enc_ipv4_addrs(rule, &match);
854ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_c,
855ea4cd837SPaul Blakey src_ipv4_src_ipv6.ipv4_layout.ipv4,
856ea4cd837SPaul Blakey ntohl(match.mask->src));
857ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_v,
858ea4cd837SPaul Blakey src_ipv4_src_ipv6.ipv4_layout.ipv4,
859ea4cd837SPaul Blakey ntohl(match.key->src));
860ea4cd837SPaul Blakey
861ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_c,
862ea4cd837SPaul Blakey dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
863ea4cd837SPaul Blakey ntohl(match.mask->dst));
864ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_v,
865ea4cd837SPaul Blakey dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
866ea4cd837SPaul Blakey ntohl(match.key->dst));
867ea4cd837SPaul Blakey
8684a5d5d73SEli Britstein key_basic.n_proto = htons(ETH_P_IP);
869fca53304SEli Britstein mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
870fca53304SEli Britstein headers_c, headers_v);
871ea4cd837SPaul Blakey } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
872ea4cd837SPaul Blakey struct flow_match_ipv6_addrs match;
873ea4cd837SPaul Blakey
874ea4cd837SPaul Blakey flow_rule_match_enc_ipv6_addrs(rule, &match);
875ea4cd837SPaul Blakey memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
876ea4cd837SPaul Blakey src_ipv4_src_ipv6.ipv6_layout.ipv6),
877ea4cd837SPaul Blakey &match.mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
878ea4cd837SPaul Blakey ipv6));
879ea4cd837SPaul Blakey memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
880ea4cd837SPaul Blakey src_ipv4_src_ipv6.ipv6_layout.ipv6),
881ea4cd837SPaul Blakey &match.key->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
882ea4cd837SPaul Blakey ipv6));
883ea4cd837SPaul Blakey
884ea4cd837SPaul Blakey memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
885ea4cd837SPaul Blakey dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
886ea4cd837SPaul Blakey &match.mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
887ea4cd837SPaul Blakey ipv6));
888ea4cd837SPaul Blakey memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
889ea4cd837SPaul Blakey dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
890ea4cd837SPaul Blakey &match.key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
891ea4cd837SPaul Blakey ipv6));
892ea4cd837SPaul Blakey
8934a5d5d73SEli Britstein key_basic.n_proto = htons(ETH_P_IPV6);
894fca53304SEli Britstein mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
895fca53304SEli Britstein headers_c, headers_v);
896ea4cd837SPaul Blakey }
897ea4cd837SPaul Blakey }
898ea4cd837SPaul Blakey
899ea4cd837SPaul Blakey if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) {
900ea4cd837SPaul Blakey struct flow_match_ip match;
901ea4cd837SPaul Blakey
902ea4cd837SPaul Blakey flow_rule_match_enc_ip(rule, &match);
903ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn,
904ea4cd837SPaul Blakey match.mask->tos & 0x3);
905ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn,
906ea4cd837SPaul Blakey match.key->tos & 0x3);
907ea4cd837SPaul Blakey
908ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp,
909ea4cd837SPaul Blakey match.mask->tos >> 2);
910ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp,
911ea4cd837SPaul Blakey match.key->tos >> 2);
912ea4cd837SPaul Blakey
913ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit,
914ea4cd837SPaul Blakey match.mask->ttl);
915ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit,
916ea4cd837SPaul Blakey match.key->ttl);
917ea4cd837SPaul Blakey
918ea4cd837SPaul Blakey if (match.mask->ttl &&
919ea4cd837SPaul Blakey !MLX5_CAP_ESW_FLOWTABLE_FDB
920ea4cd837SPaul Blakey (priv->mdev,
921ea4cd837SPaul Blakey ft_field_support.outer_ipv4_ttl)) {
922ea4cd837SPaul Blakey NL_SET_ERR_MSG_MOD(extack,
923ea4cd837SPaul Blakey "Matching on TTL is not supported");
924ea4cd837SPaul Blakey err = -EOPNOTSUPP;
925ea4cd837SPaul Blakey goto out;
926ea4cd837SPaul Blakey }
927ea4cd837SPaul Blakey }
928ea4cd837SPaul Blakey
929ea4cd837SPaul Blakey /* let software handle IP fragments */
930ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
931ea4cd837SPaul Blakey MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
932ea4cd837SPaul Blakey
933ea4cd837SPaul Blakey return 0;
934ea4cd837SPaul Blakey
935d386939aSYevgeny Kliteynik out:
936d386939aSYevgeny Kliteynik return err;
937d386939aSYevgeny Kliteynik }
938d386939aSYevgeny Kliteynik
mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,void * headers_c,void * headers_v)939d386939aSYevgeny Kliteynik int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv,
940d386939aSYevgeny Kliteynik struct mlx5_flow_spec *spec,
941f9e30088SPablo Neira Ayuso struct flow_cls_offload *f,
942d386939aSYevgeny Kliteynik void *headers_c,
943d386939aSYevgeny Kliteynik void *headers_v)
944d386939aSYevgeny Kliteynik {
945f9e30088SPablo Neira Ayuso struct flow_rule *rule = flow_cls_offload_flow_rule(f);
946d386939aSYevgeny Kliteynik struct netlink_ext_ack *extack = f->common.extack;
947d386939aSYevgeny Kliteynik struct flow_match_ports enc_ports;
948d386939aSYevgeny Kliteynik
949d386939aSYevgeny Kliteynik /* Full udp dst port must be given */
950d386939aSYevgeny Kliteynik
951d386939aSYevgeny Kliteynik if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
952d386939aSYevgeny Kliteynik NL_SET_ERR_MSG_MOD(extack,
953d386939aSYevgeny Kliteynik "UDP tunnel decap filter must include enc_dst_port condition");
954d386939aSYevgeny Kliteynik netdev_warn(priv->netdev,
955d386939aSYevgeny Kliteynik "UDP tunnel decap filter must include enc_dst_port condition\n");
956101f4de9SOz Shlomo return -EOPNOTSUPP;
957101f4de9SOz Shlomo }
958d386939aSYevgeny Kliteynik
959d386939aSYevgeny Kliteynik flow_rule_match_enc_ports(rule, &enc_ports);
960d386939aSYevgeny Kliteynik
961d386939aSYevgeny Kliteynik if (memchr_inv(&enc_ports.mask->dst, 0xff,
962d386939aSYevgeny Kliteynik sizeof(enc_ports.mask->dst))) {
963d386939aSYevgeny Kliteynik NL_SET_ERR_MSG_MOD(extack,
964d386939aSYevgeny Kliteynik "UDP tunnel decap filter must match enc_dst_port fully");
965d386939aSYevgeny Kliteynik netdev_warn(priv->netdev,
966d386939aSYevgeny Kliteynik "UDP tunnel decap filter must match enc_dst_port fully\n");
967d386939aSYevgeny Kliteynik return -EOPNOTSUPP;
968d386939aSYevgeny Kliteynik }
969d386939aSYevgeny Kliteynik
970d386939aSYevgeny Kliteynik /* match on UDP protocol and dst port number */
971d386939aSYevgeny Kliteynik
972d386939aSYevgeny Kliteynik MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
973d386939aSYevgeny Kliteynik MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
974d386939aSYevgeny Kliteynik
975d386939aSYevgeny Kliteynik MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
976d386939aSYevgeny Kliteynik ntohs(enc_ports.mask->dst));
977d386939aSYevgeny Kliteynik MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
978d386939aSYevgeny Kliteynik ntohs(enc_ports.key->dst));
979d386939aSYevgeny Kliteynik
980d386939aSYevgeny Kliteynik /* UDP src port on outer header is generated by HW,
981d386939aSYevgeny Kliteynik * so it is probably a bad idea to request matching it.
982d386939aSYevgeny Kliteynik * Nonetheless, it is allowed.
983d386939aSYevgeny Kliteynik */
984d386939aSYevgeny Kliteynik
985d386939aSYevgeny Kliteynik MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
986d386939aSYevgeny Kliteynik ntohs(enc_ports.mask->src));
987d386939aSYevgeny Kliteynik MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
988d386939aSYevgeny Kliteynik ntohs(enc_ports.key->src));
989d386939aSYevgeny Kliteynik
990d386939aSYevgeny Kliteynik return 0;
991101f4de9SOz Shlomo }
992