10d9f9647SVlad Buslov // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
20d9f9647SVlad Buslov /* Copyright (c) 2021 Mellanox Technologies. */
30d9f9647SVlad Buslov
48914add2SVlad Buslov #include <net/fib_notifier.h>
55632817bSVlad Buslov #include <net/nexthop.h>
658de53c1SGavin Li #include <net/ip_tunnels.h>
70d9f9647SVlad Buslov #include "tc_tun_encap.h"
80d9f9647SVlad Buslov #include "en_tc.h"
90d9f9647SVlad Buslov #include "tc_tun.h"
100d9f9647SVlad Buslov #include "rep/tc.h"
110d9f9647SVlad Buslov #include "diag/en_tc_tracepoint.h"
120d9f9647SVlad Buslov
138914add2SVlad Buslov enum {
148914add2SVlad Buslov MLX5E_ROUTE_ENTRY_VALID = BIT(0),
158914add2SVlad Buslov };
168914add2SVlad Buslov
mlx5e_set_int_port_tunnel(struct mlx5e_priv * priv,struct mlx5_flow_attr * attr,struct mlx5e_encap_entry * e,int out_index)17100ad4e2SAriel Levkovich static int mlx5e_set_int_port_tunnel(struct mlx5e_priv *priv,
18100ad4e2SAriel Levkovich struct mlx5_flow_attr *attr,
19100ad4e2SAriel Levkovich struct mlx5e_encap_entry *e,
20100ad4e2SAriel Levkovich int out_index)
21100ad4e2SAriel Levkovich {
22100ad4e2SAriel Levkovich struct net_device *route_dev;
23100ad4e2SAriel Levkovich int err = 0;
24100ad4e2SAriel Levkovich
25100ad4e2SAriel Levkovich route_dev = dev_get_by_index(dev_net(e->out_dev), e->route_dev_ifindex);
26100ad4e2SAriel Levkovich
2706b4eac9SJianbo Liu if (!route_dev || !netif_is_ovs_master(route_dev) ||
2806b4eac9SJianbo Liu attr->parse_attr->filter_dev == e->out_dev)
29100ad4e2SAriel Levkovich goto out;
30100ad4e2SAriel Levkovich
31100ad4e2SAriel Levkovich err = mlx5e_set_fwd_to_int_port_actions(priv, attr, e->route_dev_ifindex,
32100ad4e2SAriel Levkovich MLX5E_TC_INT_PORT_EGRESS,
33100ad4e2SAriel Levkovich &attr->action, out_index);
34100ad4e2SAriel Levkovich
35100ad4e2SAriel Levkovich out:
36100ad4e2SAriel Levkovich if (route_dev)
37100ad4e2SAriel Levkovich dev_put(route_dev);
38100ad4e2SAriel Levkovich
39100ad4e2SAriel Levkovich return err;
40100ad4e2SAriel Levkovich }
41100ad4e2SAriel Levkovich
42777bb800SVlad Buslov struct mlx5e_route_key {
43777bb800SVlad Buslov int ip_version;
44777bb800SVlad Buslov union {
45777bb800SVlad Buslov __be32 v4;
46777bb800SVlad Buslov struct in6_addr v6;
47777bb800SVlad Buslov } endpoint_ip;
48777bb800SVlad Buslov };
49777bb800SVlad Buslov
50777bb800SVlad Buslov struct mlx5e_route_entry {
51777bb800SVlad Buslov struct mlx5e_route_key key;
52777bb800SVlad Buslov struct list_head encap_entries;
53777bb800SVlad Buslov struct list_head decap_flows;
548914add2SVlad Buslov u32 flags;
55777bb800SVlad Buslov struct hlist_node hlist;
56777bb800SVlad Buslov refcount_t refcnt;
578914add2SVlad Buslov int tunnel_dev_index;
58777bb800SVlad Buslov struct rcu_head rcu;
59777bb800SVlad Buslov };
60777bb800SVlad Buslov
618914add2SVlad Buslov struct mlx5e_tc_tun_encap {
628914add2SVlad Buslov struct mlx5e_priv *priv;
638914add2SVlad Buslov struct notifier_block fib_nb;
648914add2SVlad Buslov spinlock_t route_lock; /* protects route_tbl */
658914add2SVlad Buslov unsigned long route_tbl_last_update;
668914add2SVlad Buslov DECLARE_HASHTABLE(route_tbl, 8);
678914add2SVlad Buslov };
688914add2SVlad Buslov
mlx5e_route_entry_valid(struct mlx5e_route_entry * r)698914add2SVlad Buslov static bool mlx5e_route_entry_valid(struct mlx5e_route_entry *r)
708914add2SVlad Buslov {
718914add2SVlad Buslov return r->flags & MLX5E_ROUTE_ENTRY_VALID;
728914add2SVlad Buslov }
738914add2SVlad Buslov
mlx5e_tc_set_attr_rx_tun(struct mlx5e_tc_flow * flow,struct mlx5_flow_spec * spec)740d9f9647SVlad Buslov int mlx5e_tc_set_attr_rx_tun(struct mlx5e_tc_flow *flow,
750d9f9647SVlad Buslov struct mlx5_flow_spec *spec)
760d9f9647SVlad Buslov {
770d9f9647SVlad Buslov struct mlx5_esw_flow_attr *esw_attr = flow->attr->esw_attr;
780d9f9647SVlad Buslov struct mlx5_rx_tun_attr *tun_attr;
790d9f9647SVlad Buslov void *daddr, *saddr;
800d9f9647SVlad Buslov u8 ip_version;
810d9f9647SVlad Buslov
820d9f9647SVlad Buslov tun_attr = kvzalloc(sizeof(*tun_attr), GFP_KERNEL);
830d9f9647SVlad Buslov if (!tun_attr)
840d9f9647SVlad Buslov return -ENOMEM;
850d9f9647SVlad Buslov
860d9f9647SVlad Buslov esw_attr->rx_tun_attr = tun_attr;
870d9f9647SVlad Buslov ip_version = mlx5e_tc_get_ip_version(spec, true);
880d9f9647SVlad Buslov
890d9f9647SVlad Buslov if (ip_version == 4) {
900d9f9647SVlad Buslov daddr = MLX5_ADDR_OF(fte_match_param, spec->match_value,
910d9f9647SVlad Buslov outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
920d9f9647SVlad Buslov saddr = MLX5_ADDR_OF(fte_match_param, spec->match_value,
930d9f9647SVlad Buslov outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
940d9f9647SVlad Buslov tun_attr->dst_ip.v4 = *(__be32 *)daddr;
950d9f9647SVlad Buslov tun_attr->src_ip.v4 = *(__be32 *)saddr;
96777bb800SVlad Buslov if (!tun_attr->dst_ip.v4 || !tun_attr->src_ip.v4)
97777bb800SVlad Buslov return 0;
980d9f9647SVlad Buslov }
990d9f9647SVlad Buslov #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
1000d9f9647SVlad Buslov else if (ip_version == 6) {
1010d9f9647SVlad Buslov int ipv6_size = MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6);
1020d9f9647SVlad Buslov
1030d9f9647SVlad Buslov daddr = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1040d9f9647SVlad Buslov outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1050d9f9647SVlad Buslov saddr = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1060d9f9647SVlad Buslov outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6);
1070d9f9647SVlad Buslov memcpy(&tun_attr->dst_ip.v6, daddr, ipv6_size);
1080d9f9647SVlad Buslov memcpy(&tun_attr->src_ip.v6, saddr, ipv6_size);
1098cdc3223SKuniyuki Iwashima if (ipv6_addr_any(&tun_attr->dst_ip.v6) ||
1108cdc3223SKuniyuki Iwashima ipv6_addr_any(&tun_attr->src_ip.v6))
111777bb800SVlad Buslov return 0;
1120d9f9647SVlad Buslov }
1130d9f9647SVlad Buslov #endif
114777bb800SVlad Buslov /* Only set the flag if both src and dst ip addresses exist. They are
115777bb800SVlad Buslov * required to establish routing.
116777bb800SVlad Buslov */
117777bb800SVlad Buslov flow_flag_set(flow, TUN_RX);
1181e74152eSRoi Dayan flow->attr->tun_ip_version = ip_version;
1190d9f9647SVlad Buslov return 0;
1200d9f9647SVlad Buslov }
1210d9f9647SVlad Buslov
mlx5e_tc_flow_all_encaps_valid(struct mlx5_esw_flow_attr * esw_attr)1228914add2SVlad Buslov static bool mlx5e_tc_flow_all_encaps_valid(struct mlx5_esw_flow_attr *esw_attr)
1238914add2SVlad Buslov {
1248914add2SVlad Buslov bool all_flow_encaps_valid = true;
1258914add2SVlad Buslov int i;
1268914add2SVlad Buslov
1278914add2SVlad Buslov /* Flow can be associated with multiple encap entries.
1288914add2SVlad Buslov * Before offloading the flow verify that all of them have
1298914add2SVlad Buslov * a valid neighbour.
1308914add2SVlad Buslov */
1318914add2SVlad Buslov for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
1328914add2SVlad Buslov if (!(esw_attr->dests[i].flags & MLX5_ESW_DEST_ENCAP))
1338914add2SVlad Buslov continue;
1348914add2SVlad Buslov if (!(esw_attr->dests[i].flags & MLX5_ESW_DEST_ENCAP_VALID)) {
1358914add2SVlad Buslov all_flow_encaps_valid = false;
1368914add2SVlad Buslov break;
1378914add2SVlad Buslov }
1388914add2SVlad Buslov }
1398914add2SVlad Buslov
1408914add2SVlad Buslov return all_flow_encaps_valid;
1418914add2SVlad Buslov }
1428914add2SVlad Buslov
mlx5e_tc_encap_flows_add(struct mlx5e_priv * priv,struct mlx5e_encap_entry * e,struct list_head * flow_list)1430d9f9647SVlad Buslov void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv,
1440d9f9647SVlad Buslov struct mlx5e_encap_entry *e,
1450d9f9647SVlad Buslov struct list_head *flow_list)
1460d9f9647SVlad Buslov {
1470d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1483f3f05abSYevgeny Kliteynik struct mlx5_pkt_reformat_params reformat_params;
1490d9f9647SVlad Buslov struct mlx5_esw_flow_attr *esw_attr;
1500d9f9647SVlad Buslov struct mlx5_flow_handle *rule;
1510d9f9647SVlad Buslov struct mlx5_flow_attr *attr;
1520d9f9647SVlad Buslov struct mlx5_flow_spec *spec;
1530d9f9647SVlad Buslov struct mlx5e_tc_flow *flow;
1540d9f9647SVlad Buslov int err;
1550d9f9647SVlad Buslov
1568914add2SVlad Buslov if (e->flags & MLX5_ENCAP_ENTRY_NO_ROUTE)
1578914add2SVlad Buslov return;
1588914add2SVlad Buslov
1593f3f05abSYevgeny Kliteynik memset(&reformat_params, 0, sizeof(reformat_params));
1603f3f05abSYevgeny Kliteynik reformat_params.type = e->reformat_type;
1613f3f05abSYevgeny Kliteynik reformat_params.size = e->encap_size;
1623f3f05abSYevgeny Kliteynik reformat_params.data = e->encap_header;
1630d9f9647SVlad Buslov e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev,
1643f3f05abSYevgeny Kliteynik &reformat_params,
1650d9f9647SVlad Buslov MLX5_FLOW_NAMESPACE_FDB);
1660d9f9647SVlad Buslov if (IS_ERR(e->pkt_reformat)) {
1670d9f9647SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to offload cached encapsulation header, %lu\n",
1680d9f9647SVlad Buslov PTR_ERR(e->pkt_reformat));
1690d9f9647SVlad Buslov return;
1700d9f9647SVlad Buslov }
1710d9f9647SVlad Buslov e->flags |= MLX5_ENCAP_ENTRY_VALID;
1720d9f9647SVlad Buslov mlx5e_rep_queue_neigh_stats_work(priv);
1730d9f9647SVlad Buslov
1740d9f9647SVlad Buslov list_for_each_entry(flow, flow_list, tmp_list) {
1759a5f9cc7SRoi Dayan if (!mlx5e_is_offloaded_flow(flow) || !flow_flag_test(flow, SLOW))
1760d9f9647SVlad Buslov continue;
1770d9f9647SVlad Buslov
1788300f225SRoi Dayan spec = &flow->attr->parse_attr->spec;
1798300f225SRoi Dayan
1808300f225SRoi Dayan attr = mlx5e_tc_get_encap_attr(flow);
1818300f225SRoi Dayan esw_attr = attr->esw_attr;
182021905f8SVlad Buslov esw_attr->dests[flow->tmp_entry_index].pkt_reformat = e->pkt_reformat;
183021905f8SVlad Buslov esw_attr->dests[flow->tmp_entry_index].flags |= MLX5_ESW_DEST_ENCAP_VALID;
1848914add2SVlad Buslov
1850d9f9647SVlad Buslov /* Do not offload flows with unresolved neighbors */
1868914add2SVlad Buslov if (!mlx5e_tc_flow_all_encaps_valid(esw_attr))
1870d9f9647SVlad Buslov continue;
1888300f225SRoi Dayan
1898300f225SRoi Dayan err = mlx5e_tc_offload_flow_post_acts(flow);
1908300f225SRoi Dayan if (err) {
1918300f225SRoi Dayan mlx5_core_warn(priv->mdev, "Failed to update flow post acts, %d\n",
1928300f225SRoi Dayan err);
1938300f225SRoi Dayan continue;
1948300f225SRoi Dayan }
1958300f225SRoi Dayan
1960d9f9647SVlad Buslov /* update from slow path rule to encap rule */
1978300f225SRoi Dayan rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, flow->attr);
1980d9f9647SVlad Buslov if (IS_ERR(rule)) {
1998300f225SRoi Dayan mlx5e_tc_unoffload_flow_post_acts(flow);
2000d9f9647SVlad Buslov err = PTR_ERR(rule);
2010d9f9647SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update cached encapsulation flow, %d\n",
2020d9f9647SVlad Buslov err);
2030d9f9647SVlad Buslov continue;
2040d9f9647SVlad Buslov }
2050d9f9647SVlad Buslov
2060d9f9647SVlad Buslov mlx5e_tc_unoffload_from_slow_path(esw, flow);
2070d9f9647SVlad Buslov flow->rule[0] = rule;
2080d9f9647SVlad Buslov /* was unset when slow path rule removed */
2090d9f9647SVlad Buslov flow_flag_set(flow, OFFLOADED);
2100d9f9647SVlad Buslov }
2110d9f9647SVlad Buslov }
2120d9f9647SVlad Buslov
mlx5e_tc_encap_flows_del(struct mlx5e_priv * priv,struct mlx5e_encap_entry * e,struct list_head * flow_list)2130d9f9647SVlad Buslov void mlx5e_tc_encap_flows_del(struct mlx5e_priv *priv,
2140d9f9647SVlad Buslov struct mlx5e_encap_entry *e,
2150d9f9647SVlad Buslov struct list_head *flow_list)
2160d9f9647SVlad Buslov {
2170d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
2180d9f9647SVlad Buslov struct mlx5_esw_flow_attr *esw_attr;
2190d9f9647SVlad Buslov struct mlx5_flow_handle *rule;
2200d9f9647SVlad Buslov struct mlx5_flow_attr *attr;
2210d9f9647SVlad Buslov struct mlx5_flow_spec *spec;
2220d9f9647SVlad Buslov struct mlx5e_tc_flow *flow;
2230d9f9647SVlad Buslov int err;
2240d9f9647SVlad Buslov
2250d9f9647SVlad Buslov list_for_each_entry(flow, flow_list, tmp_list) {
2262951b2e1SChris Mi if (!mlx5e_is_offloaded_flow(flow))
2270d9f9647SVlad Buslov continue;
2288300f225SRoi Dayan
2298300f225SRoi Dayan attr = mlx5e_tc_get_encap_attr(flow);
2308300f225SRoi Dayan esw_attr = attr->esw_attr;
2310d9f9647SVlad Buslov /* mark the flow's encap dest as non-valid */
232021905f8SVlad Buslov esw_attr->dests[flow->tmp_entry_index].flags &= ~MLX5_ESW_DEST_ENCAP_VALID;
2336d942e40SRoi Dayan esw_attr->dests[flow->tmp_entry_index].pkt_reformat = NULL;
2346d942e40SRoi Dayan
2352951b2e1SChris Mi /* Clear pkt_reformat before checking slow path flag. Because
2362951b2e1SChris Mi * in next iteration, the same flow is already set slow path
2372951b2e1SChris Mi * flag, but still need to clear the pkt_reformat.
2382951b2e1SChris Mi */
2392951b2e1SChris Mi if (flow_flag_test(flow, SLOW))
2402951b2e1SChris Mi continue;
2412951b2e1SChris Mi
2426d942e40SRoi Dayan /* update from encap rule to slow path rule */
2436d942e40SRoi Dayan spec = &flow->attr->parse_attr->spec;
2446d942e40SRoi Dayan rule = mlx5e_tc_offload_to_slow_path(esw, flow, spec);
2450d9f9647SVlad Buslov
2460d9f9647SVlad Buslov if (IS_ERR(rule)) {
2470d9f9647SVlad Buslov err = PTR_ERR(rule);
2480d9f9647SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update slow path (encap) flow, %d\n",
2490d9f9647SVlad Buslov err);
2500d9f9647SVlad Buslov continue;
2510d9f9647SVlad Buslov }
2520d9f9647SVlad Buslov
2538300f225SRoi Dayan mlx5e_tc_unoffload_fdb_rules(esw, flow, flow->attr);
2548300f225SRoi Dayan mlx5e_tc_unoffload_flow_post_acts(flow);
2550d9f9647SVlad Buslov flow->rule[0] = rule;
2560d9f9647SVlad Buslov /* was unset when fast path rule removed */
2570d9f9647SVlad Buslov flow_flag_set(flow, OFFLOADED);
2580d9f9647SVlad Buslov }
2590d9f9647SVlad Buslov
2600d9f9647SVlad Buslov /* we know that the encap is valid */
2610d9f9647SVlad Buslov e->flags &= ~MLX5_ENCAP_ENTRY_VALID;
2620d9f9647SVlad Buslov mlx5_packet_reformat_dealloc(priv->mdev, e->pkt_reformat);
2636d942e40SRoi Dayan e->pkt_reformat = NULL;
2640d9f9647SVlad Buslov }
2650d9f9647SVlad Buslov
mlx5e_take_tmp_flow(struct mlx5e_tc_flow * flow,struct list_head * flow_list,int index)2668914add2SVlad Buslov static void mlx5e_take_tmp_flow(struct mlx5e_tc_flow *flow,
2678914add2SVlad Buslov struct list_head *flow_list,
2688914add2SVlad Buslov int index)
2698914add2SVlad Buslov {
270362980eaSVlad Buslov if (IS_ERR(mlx5e_flow_get(flow))) {
271362980eaSVlad Buslov /* Flow is being deleted concurrently. Wait for it to be
272362980eaSVlad Buslov * unoffloaded from hardware, otherwise deleting encap will
273362980eaSVlad Buslov * fail.
274362980eaSVlad Buslov */
275362980eaSVlad Buslov wait_for_completion(&flow->del_hw_done);
2768914add2SVlad Buslov return;
277362980eaSVlad Buslov }
2788914add2SVlad Buslov wait_for_completion(&flow->init_done);
2798914add2SVlad Buslov
2808914add2SVlad Buslov flow->tmp_entry_index = index;
2818914add2SVlad Buslov list_add(&flow->tmp_list, flow_list);
2828914add2SVlad Buslov }
2838914add2SVlad Buslov
2840d9f9647SVlad Buslov /* Takes reference to all flows attached to encap and adds the flows to
2850d9f9647SVlad Buslov * flow_list using 'tmp_list' list_head in mlx5e_tc_flow.
2860d9f9647SVlad Buslov */
mlx5e_take_all_encap_flows(struct mlx5e_encap_entry * e,struct list_head * flow_list)2870d9f9647SVlad Buslov void mlx5e_take_all_encap_flows(struct mlx5e_encap_entry *e, struct list_head *flow_list)
2880d9f9647SVlad Buslov {
2890d9f9647SVlad Buslov struct encap_flow_item *efi;
2900d9f9647SVlad Buslov struct mlx5e_tc_flow *flow;
2910d9f9647SVlad Buslov
2920d9f9647SVlad Buslov list_for_each_entry(efi, &e->flows, list) {
2930d9f9647SVlad Buslov flow = container_of(efi, struct mlx5e_tc_flow, encaps[efi->index]);
2948914add2SVlad Buslov mlx5e_take_tmp_flow(flow, flow_list, efi->index);
2950d9f9647SVlad Buslov }
2960d9f9647SVlad Buslov }
2970d9f9647SVlad Buslov
2988914add2SVlad Buslov /* Takes reference to all flows attached to route and adds the flows to
2998914add2SVlad Buslov * flow_list using 'tmp_list' list_head in mlx5e_tc_flow.
3008914add2SVlad Buslov */
mlx5e_take_all_route_decap_flows(struct mlx5e_route_entry * r,struct list_head * flow_list)3018914add2SVlad Buslov static void mlx5e_take_all_route_decap_flows(struct mlx5e_route_entry *r,
3028914add2SVlad Buslov struct list_head *flow_list)
3038914add2SVlad Buslov {
3048914add2SVlad Buslov struct mlx5e_tc_flow *flow;
3058914add2SVlad Buslov
3068914add2SVlad Buslov list_for_each_entry(flow, &r->decap_flows, decap_routes)
3078914add2SVlad Buslov mlx5e_take_tmp_flow(flow, flow_list, 0);
3088914add2SVlad Buslov }
3098914add2SVlad Buslov
310fb1a3132SVlad Buslov typedef bool (match_cb)(struct mlx5e_encap_entry *);
311fb1a3132SVlad Buslov
3120d9f9647SVlad Buslov static struct mlx5e_encap_entry *
mlx5e_get_next_matching_encap(struct mlx5e_neigh_hash_entry * nhe,struct mlx5e_encap_entry * e,match_cb match)313fb1a3132SVlad Buslov mlx5e_get_next_matching_encap(struct mlx5e_neigh_hash_entry *nhe,
314fb1a3132SVlad Buslov struct mlx5e_encap_entry *e,
315fb1a3132SVlad Buslov match_cb match)
3160d9f9647SVlad Buslov {
3170d9f9647SVlad Buslov struct mlx5e_encap_entry *next = NULL;
3180d9f9647SVlad Buslov
3190d9f9647SVlad Buslov retry:
3200d9f9647SVlad Buslov rcu_read_lock();
3210d9f9647SVlad Buslov
3220d9f9647SVlad Buslov /* find encap with non-zero reference counter value */
3230d9f9647SVlad Buslov for (next = e ?
3240d9f9647SVlad Buslov list_next_or_null_rcu(&nhe->encap_list,
3250d9f9647SVlad Buslov &e->encap_list,
3260d9f9647SVlad Buslov struct mlx5e_encap_entry,
3270d9f9647SVlad Buslov encap_list) :
3280d9f9647SVlad Buslov list_first_or_null_rcu(&nhe->encap_list,
3290d9f9647SVlad Buslov struct mlx5e_encap_entry,
3300d9f9647SVlad Buslov encap_list);
3310d9f9647SVlad Buslov next;
3320d9f9647SVlad Buslov next = list_next_or_null_rcu(&nhe->encap_list,
3330d9f9647SVlad Buslov &next->encap_list,
3340d9f9647SVlad Buslov struct mlx5e_encap_entry,
3350d9f9647SVlad Buslov encap_list))
3360d9f9647SVlad Buslov if (mlx5e_encap_take(next))
3370d9f9647SVlad Buslov break;
3380d9f9647SVlad Buslov
3390d9f9647SVlad Buslov rcu_read_unlock();
3400d9f9647SVlad Buslov
3410d9f9647SVlad Buslov /* release starting encap */
3420d9f9647SVlad Buslov if (e)
3430d9f9647SVlad Buslov mlx5e_encap_put(netdev_priv(e->out_dev), e);
3440d9f9647SVlad Buslov if (!next)
3450d9f9647SVlad Buslov return next;
3460d9f9647SVlad Buslov
3470d9f9647SVlad Buslov /* wait for encap to be fully initialized */
3480d9f9647SVlad Buslov wait_for_completion(&next->res_ready);
3490d9f9647SVlad Buslov /* continue searching if encap entry is not in valid state after completion */
350fb1a3132SVlad Buslov if (!match(next)) {
3510d9f9647SVlad Buslov e = next;
3520d9f9647SVlad Buslov goto retry;
3530d9f9647SVlad Buslov }
3540d9f9647SVlad Buslov
3550d9f9647SVlad Buslov return next;
3560d9f9647SVlad Buslov }
3570d9f9647SVlad Buslov
mlx5e_encap_valid(struct mlx5e_encap_entry * e)358fb1a3132SVlad Buslov static bool mlx5e_encap_valid(struct mlx5e_encap_entry *e)
359fb1a3132SVlad Buslov {
360fb1a3132SVlad Buslov return e->flags & MLX5_ENCAP_ENTRY_VALID;
361fb1a3132SVlad Buslov }
362fb1a3132SVlad Buslov
363fb1a3132SVlad Buslov static struct mlx5e_encap_entry *
mlx5e_get_next_valid_encap(struct mlx5e_neigh_hash_entry * nhe,struct mlx5e_encap_entry * e)364fb1a3132SVlad Buslov mlx5e_get_next_valid_encap(struct mlx5e_neigh_hash_entry *nhe,
365fb1a3132SVlad Buslov struct mlx5e_encap_entry *e)
366fb1a3132SVlad Buslov {
367fb1a3132SVlad Buslov return mlx5e_get_next_matching_encap(nhe, e, mlx5e_encap_valid);
368fb1a3132SVlad Buslov }
369fb1a3132SVlad Buslov
mlx5e_encap_initialized(struct mlx5e_encap_entry * e)370fb1a3132SVlad Buslov static bool mlx5e_encap_initialized(struct mlx5e_encap_entry *e)
371fb1a3132SVlad Buslov {
372fb1a3132SVlad Buslov return e->compl_result >= 0;
373fb1a3132SVlad Buslov }
374fb1a3132SVlad Buslov
375fb1a3132SVlad Buslov struct mlx5e_encap_entry *
mlx5e_get_next_init_encap(struct mlx5e_neigh_hash_entry * nhe,struct mlx5e_encap_entry * e)376fb1a3132SVlad Buslov mlx5e_get_next_init_encap(struct mlx5e_neigh_hash_entry *nhe,
377fb1a3132SVlad Buslov struct mlx5e_encap_entry *e)
378fb1a3132SVlad Buslov {
379fb1a3132SVlad Buslov return mlx5e_get_next_matching_encap(nhe, e, mlx5e_encap_initialized);
380fb1a3132SVlad Buslov }
381fb1a3132SVlad Buslov
mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry * nhe)3820d9f9647SVlad Buslov void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
3830d9f9647SVlad Buslov {
3840d9f9647SVlad Buslov struct mlx5e_neigh *m_neigh = &nhe->m_neigh;
3850d9f9647SVlad Buslov struct mlx5e_encap_entry *e = NULL;
3860d9f9647SVlad Buslov struct mlx5e_tc_flow *flow;
3870d9f9647SVlad Buslov struct mlx5_fc *counter;
3880d9f9647SVlad Buslov struct neigh_table *tbl;
3890d9f9647SVlad Buslov bool neigh_used = false;
3900d9f9647SVlad Buslov struct neighbour *n;
3910d9f9647SVlad Buslov u64 lastuse;
3920d9f9647SVlad Buslov
3930d9f9647SVlad Buslov if (m_neigh->family == AF_INET)
3940d9f9647SVlad Buslov tbl = &arp_tbl;
3950d9f9647SVlad Buslov #if IS_ENABLED(CONFIG_IPV6)
3960d9f9647SVlad Buslov else if (m_neigh->family == AF_INET6)
3970d9f9647SVlad Buslov tbl = ipv6_stub->nd_tbl;
3980d9f9647SVlad Buslov #endif
3990d9f9647SVlad Buslov else
4000d9f9647SVlad Buslov return;
4010d9f9647SVlad Buslov
4020d9f9647SVlad Buslov /* mlx5e_get_next_valid_encap() releases previous encap before returning
4030d9f9647SVlad Buslov * next one.
4040d9f9647SVlad Buslov */
4050d9f9647SVlad Buslov while ((e = mlx5e_get_next_valid_encap(nhe, e)) != NULL) {
4060d9f9647SVlad Buslov struct mlx5e_priv *priv = netdev_priv(e->out_dev);
4070d9f9647SVlad Buslov struct encap_flow_item *efi, *tmp;
4080d9f9647SVlad Buslov struct mlx5_eswitch *esw;
4090d9f9647SVlad Buslov LIST_HEAD(flow_list);
4100d9f9647SVlad Buslov
4110d9f9647SVlad Buslov esw = priv->mdev->priv.eswitch;
4120d9f9647SVlad Buslov mutex_lock(&esw->offloads.encap_tbl_lock);
4130d9f9647SVlad Buslov list_for_each_entry_safe(efi, tmp, &e->flows, list) {
4140d9f9647SVlad Buslov flow = container_of(efi, struct mlx5e_tc_flow,
4150d9f9647SVlad Buslov encaps[efi->index]);
4160d9f9647SVlad Buslov if (IS_ERR(mlx5e_flow_get(flow)))
4170d9f9647SVlad Buslov continue;
4180d9f9647SVlad Buslov list_add(&flow->tmp_list, &flow_list);
4190d9f9647SVlad Buslov
4200d9f9647SVlad Buslov if (mlx5e_is_offloaded_flow(flow)) {
4210d9f9647SVlad Buslov counter = mlx5e_tc_get_counter(flow);
4220d9f9647SVlad Buslov lastuse = mlx5_fc_query_lastuse(counter);
4230d9f9647SVlad Buslov if (time_after((unsigned long)lastuse, nhe->reported_lastuse)) {
4240d9f9647SVlad Buslov neigh_used = true;
4250d9f9647SVlad Buslov break;
4260d9f9647SVlad Buslov }
4270d9f9647SVlad Buslov }
4280d9f9647SVlad Buslov }
4290d9f9647SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
4300d9f9647SVlad Buslov
431021905f8SVlad Buslov mlx5e_put_flow_list(priv, &flow_list);
4320d9f9647SVlad Buslov if (neigh_used) {
4330d9f9647SVlad Buslov /* release current encap before breaking the loop */
4340d9f9647SVlad Buslov mlx5e_encap_put(priv, e);
4350d9f9647SVlad Buslov break;
4360d9f9647SVlad Buslov }
4370d9f9647SVlad Buslov }
4380d9f9647SVlad Buslov
4390d9f9647SVlad Buslov trace_mlx5e_tc_update_neigh_used_value(nhe, neigh_used);
4400d9f9647SVlad Buslov
4410d9f9647SVlad Buslov if (neigh_used) {
4420d9f9647SVlad Buslov nhe->reported_lastuse = jiffies;
4430d9f9647SVlad Buslov
4440d9f9647SVlad Buslov /* find the relevant neigh according to the cached device and
4450d9f9647SVlad Buslov * dst ip pair
4460d9f9647SVlad Buslov */
4472221d954SVlad Buslov n = neigh_lookup(tbl, &m_neigh->dst_ip, READ_ONCE(nhe->neigh_dev));
4480d9f9647SVlad Buslov if (!n)
4490d9f9647SVlad Buslov return;
4500d9f9647SVlad Buslov
4510d9f9647SVlad Buslov neigh_event_send(n, NULL);
4520d9f9647SVlad Buslov neigh_release(n);
4530d9f9647SVlad Buslov }
4540d9f9647SVlad Buslov }
4550d9f9647SVlad Buslov
mlx5e_encap_dealloc(struct mlx5e_priv * priv,struct mlx5e_encap_entry * e)4560d9f9647SVlad Buslov static void mlx5e_encap_dealloc(struct mlx5e_priv *priv, struct mlx5e_encap_entry *e)
4570d9f9647SVlad Buslov {
4580d9f9647SVlad Buslov WARN_ON(!list_empty(&e->flows));
4590d9f9647SVlad Buslov
4600d9f9647SVlad Buslov if (e->compl_result > 0) {
4610d9f9647SVlad Buslov mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
4620d9f9647SVlad Buslov
4630d9f9647SVlad Buslov if (e->flags & MLX5_ENCAP_ENTRY_VALID)
4640d9f9647SVlad Buslov mlx5_packet_reformat_dealloc(priv->mdev, e->pkt_reformat);
4650d9f9647SVlad Buslov }
4660d9f9647SVlad Buslov
4670d9f9647SVlad Buslov kfree(e->tun_info);
4680d9f9647SVlad Buslov kfree(e->encap_header);
4690d9f9647SVlad Buslov kfree_rcu(e, rcu);
4700d9f9647SVlad Buslov }
4710d9f9647SVlad Buslov
mlx5e_decap_dealloc(struct mlx5e_priv * priv,struct mlx5e_decap_entry * d)4720d9f9647SVlad Buslov static void mlx5e_decap_dealloc(struct mlx5e_priv *priv,
4730d9f9647SVlad Buslov struct mlx5e_decap_entry *d)
4740d9f9647SVlad Buslov {
4750d9f9647SVlad Buslov WARN_ON(!list_empty(&d->flows));
4760d9f9647SVlad Buslov
4770d9f9647SVlad Buslov if (!d->compl_result)
4780d9f9647SVlad Buslov mlx5_packet_reformat_dealloc(priv->mdev, d->pkt_reformat);
4790d9f9647SVlad Buslov
4800d9f9647SVlad Buslov kfree_rcu(d, rcu);
4810d9f9647SVlad Buslov }
4820d9f9647SVlad Buslov
mlx5e_encap_put(struct mlx5e_priv * priv,struct mlx5e_encap_entry * e)4830d9f9647SVlad Buslov void mlx5e_encap_put(struct mlx5e_priv *priv, struct mlx5e_encap_entry *e)
4840d9f9647SVlad Buslov {
4850d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
4860d9f9647SVlad Buslov
4870d9f9647SVlad Buslov if (!refcount_dec_and_mutex_lock(&e->refcnt, &esw->offloads.encap_tbl_lock))
4880d9f9647SVlad Buslov return;
489777bb800SVlad Buslov list_del(&e->route_list);
4900d9f9647SVlad Buslov hash_del_rcu(&e->encap_hlist);
4910d9f9647SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
4920d9f9647SVlad Buslov
4930d9f9647SVlad Buslov mlx5e_encap_dealloc(priv, e);
4940d9f9647SVlad Buslov }
4950d9f9647SVlad Buslov
mlx5e_encap_put_locked(struct mlx5e_priv * priv,struct mlx5e_encap_entry * e)49637c3b9faSChris Mi static void mlx5e_encap_put_locked(struct mlx5e_priv *priv, struct mlx5e_encap_entry *e)
49737c3b9faSChris Mi {
49837c3b9faSChris Mi struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
49937c3b9faSChris Mi
50037c3b9faSChris Mi lockdep_assert_held(&esw->offloads.encap_tbl_lock);
50137c3b9faSChris Mi
50237c3b9faSChris Mi if (!refcount_dec_and_test(&e->refcnt))
50337c3b9faSChris Mi return;
50437c3b9faSChris Mi list_del(&e->route_list);
50537c3b9faSChris Mi hash_del_rcu(&e->encap_hlist);
50637c3b9faSChris Mi mlx5e_encap_dealloc(priv, e);
50737c3b9faSChris Mi }
50837c3b9faSChris Mi
mlx5e_decap_put(struct mlx5e_priv * priv,struct mlx5e_decap_entry * d)5090d9f9647SVlad Buslov static void mlx5e_decap_put(struct mlx5e_priv *priv, struct mlx5e_decap_entry *d)
5100d9f9647SVlad Buslov {
5110d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
5120d9f9647SVlad Buslov
5130d9f9647SVlad Buslov if (!refcount_dec_and_mutex_lock(&d->refcnt, &esw->offloads.decap_tbl_lock))
5140d9f9647SVlad Buslov return;
5150d9f9647SVlad Buslov hash_del_rcu(&d->hlist);
5160d9f9647SVlad Buslov mutex_unlock(&esw->offloads.decap_tbl_lock);
5170d9f9647SVlad Buslov
5180d9f9647SVlad Buslov mlx5e_decap_dealloc(priv, d);
5190d9f9647SVlad Buslov }
5200d9f9647SVlad Buslov
521777bb800SVlad Buslov static void mlx5e_detach_encap_route(struct mlx5e_priv *priv,
522777bb800SVlad Buslov struct mlx5e_tc_flow *flow,
523777bb800SVlad Buslov int out_index);
524777bb800SVlad Buslov
mlx5e_detach_encap(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow,struct mlx5_flow_attr * attr,int out_index)5250d9f9647SVlad Buslov void mlx5e_detach_encap(struct mlx5e_priv *priv,
526c118ebc9SRoi Dayan struct mlx5e_tc_flow *flow,
527c118ebc9SRoi Dayan struct mlx5_flow_attr *attr,
528c118ebc9SRoi Dayan int out_index)
5290d9f9647SVlad Buslov {
5300d9f9647SVlad Buslov struct mlx5e_encap_entry *e = flow->encaps[out_index].e;
5310d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
5320d9f9647SVlad Buslov
5338300f225SRoi Dayan if (!mlx5e_is_eswitch_flow(flow))
5348300f225SRoi Dayan return;
5358300f225SRoi Dayan
536c118ebc9SRoi Dayan if (attr->esw_attr->dests[out_index].flags &
537777bb800SVlad Buslov MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
538777bb800SVlad Buslov mlx5e_detach_encap_route(priv, flow, out_index);
539777bb800SVlad Buslov
5400d9f9647SVlad Buslov /* flow wasn't fully initialized */
5410d9f9647SVlad Buslov if (!e)
5420d9f9647SVlad Buslov return;
5430d9f9647SVlad Buslov
5440d9f9647SVlad Buslov mutex_lock(&esw->offloads.encap_tbl_lock);
5450d9f9647SVlad Buslov list_del(&flow->encaps[out_index].list);
5460d9f9647SVlad Buslov flow->encaps[out_index].e = NULL;
5470d9f9647SVlad Buslov if (!refcount_dec_and_test(&e->refcnt)) {
5480d9f9647SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
5490d9f9647SVlad Buslov return;
5500d9f9647SVlad Buslov }
551777bb800SVlad Buslov list_del(&e->route_list);
5520d9f9647SVlad Buslov hash_del_rcu(&e->encap_hlist);
5530d9f9647SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
5540d9f9647SVlad Buslov
5550d9f9647SVlad Buslov mlx5e_encap_dealloc(priv, e);
5560d9f9647SVlad Buslov }
5570d9f9647SVlad Buslov
mlx5e_detach_decap(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow)5580d9f9647SVlad Buslov void mlx5e_detach_decap(struct mlx5e_priv *priv,
5590d9f9647SVlad Buslov struct mlx5e_tc_flow *flow)
5600d9f9647SVlad Buslov {
5610d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
5620d9f9647SVlad Buslov struct mlx5e_decap_entry *d = flow->decap_reformat;
5630d9f9647SVlad Buslov
5640d9f9647SVlad Buslov if (!d)
5650d9f9647SVlad Buslov return;
5660d9f9647SVlad Buslov
5670d9f9647SVlad Buslov mutex_lock(&esw->offloads.decap_tbl_lock);
5680d9f9647SVlad Buslov list_del(&flow->l3_to_l2_reformat);
5690d9f9647SVlad Buslov flow->decap_reformat = NULL;
5700d9f9647SVlad Buslov
5710d9f9647SVlad Buslov if (!refcount_dec_and_test(&d->refcnt)) {
5720d9f9647SVlad Buslov mutex_unlock(&esw->offloads.decap_tbl_lock);
5730d9f9647SVlad Buslov return;
5740d9f9647SVlad Buslov }
5750d9f9647SVlad Buslov hash_del_rcu(&d->hlist);
5760d9f9647SVlad Buslov mutex_unlock(&esw->offloads.decap_tbl_lock);
5770d9f9647SVlad Buslov
5780d9f9647SVlad Buslov mlx5e_decap_dealloc(priv, d);
5790d9f9647SVlad Buslov }
5800d9f9647SVlad Buslov
mlx5e_tc_tun_encap_info_equal_generic(struct mlx5e_encap_key * a,struct mlx5e_encap_key * b)581929a2fadSDima Chumak bool mlx5e_tc_tun_encap_info_equal_generic(struct mlx5e_encap_key *a,
582929a2fadSDima Chumak struct mlx5e_encap_key *b)
5830d9f9647SVlad Buslov {
584929a2fadSDima Chumak return memcmp(a->ip_tun_key, b->ip_tun_key, sizeof(*a->ip_tun_key)) == 0 &&
585929a2fadSDima Chumak a->tc_tunnel->tunnel_type == b->tc_tunnel->tunnel_type;
5860d9f9647SVlad Buslov }
5870d9f9647SVlad Buslov
mlx5e_tc_tun_encap_info_equal_options(struct mlx5e_encap_key * a,struct mlx5e_encap_key * b,__be16 tun_flags)58858de53c1SGavin Li bool mlx5e_tc_tun_encap_info_equal_options(struct mlx5e_encap_key *a,
58958de53c1SGavin Li struct mlx5e_encap_key *b,
59058de53c1SGavin Li __be16 tun_flags)
59158de53c1SGavin Li {
59258de53c1SGavin Li struct ip_tunnel_info *a_info;
59358de53c1SGavin Li struct ip_tunnel_info *b_info;
59458de53c1SGavin Li bool a_has_opts, b_has_opts;
59558de53c1SGavin Li
59658de53c1SGavin Li if (!mlx5e_tc_tun_encap_info_equal_generic(a, b))
59758de53c1SGavin Li return false;
59858de53c1SGavin Li
59958de53c1SGavin Li a_has_opts = !!(a->ip_tun_key->tun_flags & tun_flags);
60058de53c1SGavin Li b_has_opts = !!(b->ip_tun_key->tun_flags & tun_flags);
60158de53c1SGavin Li
60258de53c1SGavin Li /* keys are equal when both don't have any options attached */
60358de53c1SGavin Li if (!a_has_opts && !b_has_opts)
60458de53c1SGavin Li return true;
60558de53c1SGavin Li
60658de53c1SGavin Li if (a_has_opts != b_has_opts)
60758de53c1SGavin Li return false;
60858de53c1SGavin Li
60958de53c1SGavin Li /* options stored in memory next to ip_tunnel_info struct */
61058de53c1SGavin Li a_info = container_of(a->ip_tun_key, struct ip_tunnel_info, key);
61158de53c1SGavin Li b_info = container_of(b->ip_tun_key, struct ip_tunnel_info, key);
61258de53c1SGavin Li
61358de53c1SGavin Li return a_info->options_len == b_info->options_len &&
61458de53c1SGavin Li !memcmp(ip_tunnel_info_opts(a_info),
61558de53c1SGavin Li ip_tunnel_info_opts(b_info),
61658de53c1SGavin Li a_info->options_len);
61758de53c1SGavin Li }
61858de53c1SGavin Li
cmp_decap_info(struct mlx5e_decap_key * a,struct mlx5e_decap_key * b)6190d9f9647SVlad Buslov static int cmp_decap_info(struct mlx5e_decap_key *a,
6200d9f9647SVlad Buslov struct mlx5e_decap_key *b)
6210d9f9647SVlad Buslov {
6220d9f9647SVlad Buslov return memcmp(&a->key, &b->key, sizeof(b->key));
6230d9f9647SVlad Buslov }
6240d9f9647SVlad Buslov
hash_encap_info(struct mlx5e_encap_key * key)625929a2fadSDima Chumak static int hash_encap_info(struct mlx5e_encap_key *key)
6260d9f9647SVlad Buslov {
6270d9f9647SVlad Buslov return jhash(key->ip_tun_key, sizeof(*key->ip_tun_key),
6280d9f9647SVlad Buslov key->tc_tunnel->tunnel_type);
6290d9f9647SVlad Buslov }
6300d9f9647SVlad Buslov
hash_decap_info(struct mlx5e_decap_key * key)6310d9f9647SVlad Buslov static int hash_decap_info(struct mlx5e_decap_key *key)
6320d9f9647SVlad Buslov {
6330d9f9647SVlad Buslov return jhash(&key->key, sizeof(key->key), 0);
6340d9f9647SVlad Buslov }
6350d9f9647SVlad Buslov
mlx5e_encap_take(struct mlx5e_encap_entry * e)6360d9f9647SVlad Buslov bool mlx5e_encap_take(struct mlx5e_encap_entry *e)
6370d9f9647SVlad Buslov {
6380d9f9647SVlad Buslov return refcount_inc_not_zero(&e->refcnt);
6390d9f9647SVlad Buslov }
6400d9f9647SVlad Buslov
mlx5e_decap_take(struct mlx5e_decap_entry * e)6410d9f9647SVlad Buslov static bool mlx5e_decap_take(struct mlx5e_decap_entry *e)
6420d9f9647SVlad Buslov {
6430d9f9647SVlad Buslov return refcount_inc_not_zero(&e->refcnt);
6440d9f9647SVlad Buslov }
6450d9f9647SVlad Buslov
6460d9f9647SVlad Buslov static struct mlx5e_encap_entry *
mlx5e_encap_get(struct mlx5e_priv * priv,struct mlx5e_encap_key * key,uintptr_t hash_key)647929a2fadSDima Chumak mlx5e_encap_get(struct mlx5e_priv *priv, struct mlx5e_encap_key *key,
6480d9f9647SVlad Buslov uintptr_t hash_key)
6490d9f9647SVlad Buslov {
6500d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
651929a2fadSDima Chumak struct mlx5e_encap_key e_key;
6520d9f9647SVlad Buslov struct mlx5e_encap_entry *e;
6530d9f9647SVlad Buslov
6540d9f9647SVlad Buslov hash_for_each_possible_rcu(esw->offloads.encap_tbl, e,
6550d9f9647SVlad Buslov encap_hlist, hash_key) {
6560d9f9647SVlad Buslov e_key.ip_tun_key = &e->tun_info->key;
6570d9f9647SVlad Buslov e_key.tc_tunnel = e->tunnel;
658929a2fadSDima Chumak if (e->tunnel->encap_info_equal(&e_key, key) &&
6590d9f9647SVlad Buslov mlx5e_encap_take(e))
6600d9f9647SVlad Buslov return e;
6610d9f9647SVlad Buslov }
6620d9f9647SVlad Buslov
6630d9f9647SVlad Buslov return NULL;
6640d9f9647SVlad Buslov }
6650d9f9647SVlad Buslov
6660d9f9647SVlad Buslov static struct mlx5e_decap_entry *
mlx5e_decap_get(struct mlx5e_priv * priv,struct mlx5e_decap_key * key,uintptr_t hash_key)6670d9f9647SVlad Buslov mlx5e_decap_get(struct mlx5e_priv *priv, struct mlx5e_decap_key *key,
6680d9f9647SVlad Buslov uintptr_t hash_key)
6690d9f9647SVlad Buslov {
6700d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
6710d9f9647SVlad Buslov struct mlx5e_decap_key r_key;
6720d9f9647SVlad Buslov struct mlx5e_decap_entry *e;
6730d9f9647SVlad Buslov
6740d9f9647SVlad Buslov hash_for_each_possible_rcu(esw->offloads.decap_tbl, e,
6750d9f9647SVlad Buslov hlist, hash_key) {
6760d9f9647SVlad Buslov r_key = e->key;
6770d9f9647SVlad Buslov if (!cmp_decap_info(&r_key, key) &&
6780d9f9647SVlad Buslov mlx5e_decap_take(e))
6790d9f9647SVlad Buslov return e;
6800d9f9647SVlad Buslov }
6810d9f9647SVlad Buslov return NULL;
6820d9f9647SVlad Buslov }
6830d9f9647SVlad Buslov
mlx5e_dup_tun_info(const struct ip_tunnel_info * tun_info)6840d9f9647SVlad Buslov struct ip_tunnel_info *mlx5e_dup_tun_info(const struct ip_tunnel_info *tun_info)
6850d9f9647SVlad Buslov {
6860d9f9647SVlad Buslov size_t tun_size = sizeof(*tun_info) + tun_info->options_len;
6870d9f9647SVlad Buslov
6880d9f9647SVlad Buslov return kmemdup(tun_info, tun_size, GFP_KERNEL);
6890d9f9647SVlad Buslov }
6900d9f9647SVlad Buslov
is_duplicated_encap_entry(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow,int out_index,struct mlx5e_encap_entry * e,struct netlink_ext_ack * extack)6910d9f9647SVlad Buslov static bool is_duplicated_encap_entry(struct mlx5e_priv *priv,
6920d9f9647SVlad Buslov struct mlx5e_tc_flow *flow,
6930d9f9647SVlad Buslov int out_index,
6940d9f9647SVlad Buslov struct mlx5e_encap_entry *e,
6950d9f9647SVlad Buslov struct netlink_ext_ack *extack)
6960d9f9647SVlad Buslov {
6970d9f9647SVlad Buslov int i;
6980d9f9647SVlad Buslov
6990d9f9647SVlad Buslov for (i = 0; i < out_index; i++) {
7000d9f9647SVlad Buslov if (flow->encaps[i].e != e)
7010d9f9647SVlad Buslov continue;
7020d9f9647SVlad Buslov NL_SET_ERR_MSG_MOD(extack, "can't duplicate encap action");
7030d9f9647SVlad Buslov netdev_err(priv->netdev, "can't duplicate encap action\n");
7040d9f9647SVlad Buslov return true;
7050d9f9647SVlad Buslov }
7060d9f9647SVlad Buslov
7070d9f9647SVlad Buslov return false;
7080d9f9647SVlad Buslov }
7090d9f9647SVlad Buslov
mlx5e_set_vf_tunnel(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5e_tc_mod_hdr_acts * mod_hdr_acts,struct net_device * out_dev,int route_dev_ifindex,int out_index)7100d9f9647SVlad Buslov static int mlx5e_set_vf_tunnel(struct mlx5_eswitch *esw,
7110d9f9647SVlad Buslov struct mlx5_flow_attr *attr,
7120d9f9647SVlad Buslov struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts,
7130d9f9647SVlad Buslov struct net_device *out_dev,
7140d9f9647SVlad Buslov int route_dev_ifindex,
7150d9f9647SVlad Buslov int out_index)
7160d9f9647SVlad Buslov {
7170d9f9647SVlad Buslov struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
7180d9f9647SVlad Buslov struct net_device *route_dev;
7190d9f9647SVlad Buslov u16 vport_num;
7200d9f9647SVlad Buslov int err = 0;
7210d9f9647SVlad Buslov u32 data;
7220d9f9647SVlad Buslov
7230d9f9647SVlad Buslov route_dev = dev_get_by_index(dev_net(out_dev), route_dev_ifindex);
7240d9f9647SVlad Buslov
7250d9f9647SVlad Buslov if (!route_dev || route_dev->netdev_ops != &mlx5e_netdev_ops ||
7260d9f9647SVlad Buslov !mlx5e_tc_is_vf_tunnel(out_dev, route_dev))
7270d9f9647SVlad Buslov goto out;
7280d9f9647SVlad Buslov
7290d9f9647SVlad Buslov err = mlx5e_tc_query_route_vport(out_dev, route_dev, &vport_num);
7300d9f9647SVlad Buslov if (err)
7310d9f9647SVlad Buslov goto out;
7320d9f9647SVlad Buslov
7330d9f9647SVlad Buslov attr->dest_chain = 0;
7340d9f9647SVlad Buslov attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
7350d9f9647SVlad Buslov esw_attr->dests[out_index].flags |= MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE;
7360d9f9647SVlad Buslov data = mlx5_eswitch_get_vport_metadata_for_set(esw_attr->in_mdev->priv.eswitch,
7370d9f9647SVlad Buslov vport_num);
7388914add2SVlad Buslov err = mlx5e_tc_match_to_reg_set_and_get_id(esw->dev, mod_hdr_acts,
7398914add2SVlad Buslov MLX5_FLOW_NAMESPACE_FDB,
7408914add2SVlad Buslov VPORT_TO_REG, data);
7418914add2SVlad Buslov if (err >= 0) {
7428914add2SVlad Buslov esw_attr->dests[out_index].src_port_rewrite_act_id = err;
7438914add2SVlad Buslov err = 0;
7448914add2SVlad Buslov }
7450d9f9647SVlad Buslov
7460d9f9647SVlad Buslov out:
7470d9f9647SVlad Buslov if (route_dev)
7480d9f9647SVlad Buslov dev_put(route_dev);
7490d9f9647SVlad Buslov return err;
7500d9f9647SVlad Buslov }
7510d9f9647SVlad Buslov
mlx5e_update_vf_tunnel(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * attr,struct mlx5e_tc_mod_hdr_acts * mod_hdr_acts,struct net_device * out_dev,int route_dev_ifindex,int out_index)7528914add2SVlad Buslov static int mlx5e_update_vf_tunnel(struct mlx5_eswitch *esw,
7538914add2SVlad Buslov struct mlx5_esw_flow_attr *attr,
7548914add2SVlad Buslov struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts,
7558914add2SVlad Buslov struct net_device *out_dev,
7568914add2SVlad Buslov int route_dev_ifindex,
7578914add2SVlad Buslov int out_index)
7588914add2SVlad Buslov {
7598914add2SVlad Buslov int act_id = attr->dests[out_index].src_port_rewrite_act_id;
7608914add2SVlad Buslov struct net_device *route_dev;
7618914add2SVlad Buslov u16 vport_num;
7628914add2SVlad Buslov int err = 0;
7638914add2SVlad Buslov u32 data;
7648914add2SVlad Buslov
7658914add2SVlad Buslov route_dev = dev_get_by_index(dev_net(out_dev), route_dev_ifindex);
7668914add2SVlad Buslov
7678914add2SVlad Buslov if (!route_dev || route_dev->netdev_ops != &mlx5e_netdev_ops ||
7688914add2SVlad Buslov !mlx5e_tc_is_vf_tunnel(out_dev, route_dev)) {
7698914add2SVlad Buslov err = -ENODEV;
7708914add2SVlad Buslov goto out;
7718914add2SVlad Buslov }
7728914add2SVlad Buslov
7738914add2SVlad Buslov err = mlx5e_tc_query_route_vport(out_dev, route_dev, &vport_num);
7748914add2SVlad Buslov if (err)
7758914add2SVlad Buslov goto out;
7768914add2SVlad Buslov
7778914add2SVlad Buslov data = mlx5_eswitch_get_vport_metadata_for_set(attr->in_mdev->priv.eswitch,
7788914add2SVlad Buslov vport_num);
7798914add2SVlad Buslov mlx5e_tc_match_to_reg_mod_hdr_change(esw->dev, mod_hdr_acts, VPORT_TO_REG, act_id, data);
7808914add2SVlad Buslov
7818914add2SVlad Buslov out:
7828914add2SVlad Buslov if (route_dev)
7838914add2SVlad Buslov dev_put(route_dev);
7848914add2SVlad Buslov return err;
7858914add2SVlad Buslov }
7868914add2SVlad Buslov
mlx5e_route_tbl_get_last_update(struct mlx5e_priv * priv)7878914add2SVlad Buslov static unsigned int mlx5e_route_tbl_get_last_update(struct mlx5e_priv *priv)
7888914add2SVlad Buslov {
7898914add2SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
7908914add2SVlad Buslov struct mlx5_rep_uplink_priv *uplink_priv;
7918914add2SVlad Buslov struct mlx5e_rep_priv *uplink_rpriv;
7928914add2SVlad Buslov struct mlx5e_tc_tun_encap *encap;
7938914add2SVlad Buslov unsigned int ret;
7948914add2SVlad Buslov
7958914add2SVlad Buslov uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
7968914add2SVlad Buslov uplink_priv = &uplink_rpriv->uplink_priv;
7978914add2SVlad Buslov encap = uplink_priv->encap;
7988914add2SVlad Buslov
7998914add2SVlad Buslov spin_lock_bh(&encap->route_lock);
8008914add2SVlad Buslov ret = encap->route_tbl_last_update;
8018914add2SVlad Buslov spin_unlock_bh(&encap->route_lock);
8028914add2SVlad Buslov return ret;
8038914add2SVlad Buslov }
8048914add2SVlad Buslov
805777bb800SVlad Buslov static int mlx5e_attach_encap_route(struct mlx5e_priv *priv,
806777bb800SVlad Buslov struct mlx5e_tc_flow *flow,
807c118ebc9SRoi Dayan struct mlx5_flow_attr *attr,
808777bb800SVlad Buslov struct mlx5e_encap_entry *e,
809777bb800SVlad Buslov bool new_encap_entry,
8108914add2SVlad Buslov unsigned long tbl_time_before,
811777bb800SVlad Buslov int out_index);
812777bb800SVlad Buslov
mlx5e_attach_encap(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow,struct mlx5_flow_attr * attr,struct net_device * mirred_dev,int out_index,struct netlink_ext_ack * extack,struct net_device ** encap_dev)8130d9f9647SVlad Buslov int mlx5e_attach_encap(struct mlx5e_priv *priv,
8140d9f9647SVlad Buslov struct mlx5e_tc_flow *flow,
815c118ebc9SRoi Dayan struct mlx5_flow_attr *attr,
8160d9f9647SVlad Buslov struct net_device *mirred_dev,
8170d9f9647SVlad Buslov int out_index,
8180d9f9647SVlad Buslov struct netlink_ext_ack *extack,
819f3774220SChris Mi struct net_device **encap_dev)
8200d9f9647SVlad Buslov {
8210d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
8220d9f9647SVlad Buslov struct mlx5e_tc_flow_parse_attr *parse_attr;
8230d9f9647SVlad Buslov const struct ip_tunnel_info *tun_info;
824c63741b4SMaor Dickman const struct mlx5e_mpls_info *mpls_info;
8258914add2SVlad Buslov unsigned long tbl_time_before = 0;
8260d9f9647SVlad Buslov struct mlx5e_encap_entry *e;
827929a2fadSDima Chumak struct mlx5e_encap_key key;
828777bb800SVlad Buslov bool entry_created = false;
8290d9f9647SVlad Buslov unsigned short family;
8300d9f9647SVlad Buslov uintptr_t hash_key;
8310d9f9647SVlad Buslov int err = 0;
8320d9f9647SVlad Buslov
83337c3b9faSChris Mi lockdep_assert_held(&esw->offloads.encap_tbl_lock);
83437c3b9faSChris Mi
8350d9f9647SVlad Buslov parse_attr = attr->parse_attr;
8360d9f9647SVlad Buslov tun_info = parse_attr->tun_info[out_index];
837c63741b4SMaor Dickman mpls_info = &parse_attr->mpls_info[out_index];
8380d9f9647SVlad Buslov family = ip_tunnel_info_af(tun_info);
8390d9f9647SVlad Buslov key.ip_tun_key = &tun_info->key;
8400d9f9647SVlad Buslov key.tc_tunnel = mlx5e_get_tc_tun(mirred_dev);
8410d9f9647SVlad Buslov if (!key.tc_tunnel) {
8420d9f9647SVlad Buslov NL_SET_ERR_MSG_MOD(extack, "Unsupported tunnel");
8430d9f9647SVlad Buslov return -EOPNOTSUPP;
8440d9f9647SVlad Buslov }
8450d9f9647SVlad Buslov
8460d9f9647SVlad Buslov hash_key = hash_encap_info(&key);
8470d9f9647SVlad Buslov
8480d9f9647SVlad Buslov e = mlx5e_encap_get(priv, &key, hash_key);
8490d9f9647SVlad Buslov
8500d9f9647SVlad Buslov /* must verify if encap is valid or not */
8510d9f9647SVlad Buslov if (e) {
8520d9f9647SVlad Buslov /* Check that entry was not already attached to this flow */
8530d9f9647SVlad Buslov if (is_duplicated_encap_entry(priv, flow, out_index, e, extack)) {
8540d9f9647SVlad Buslov err = -EOPNOTSUPP;
8550d9f9647SVlad Buslov goto out_err;
8560d9f9647SVlad Buslov }
8570d9f9647SVlad Buslov
8580d9f9647SVlad Buslov goto attach_flow;
8590d9f9647SVlad Buslov }
8600d9f9647SVlad Buslov
8610d9f9647SVlad Buslov e = kzalloc(sizeof(*e), GFP_KERNEL);
8620d9f9647SVlad Buslov if (!e) {
8630d9f9647SVlad Buslov err = -ENOMEM;
8640d9f9647SVlad Buslov goto out_err;
8650d9f9647SVlad Buslov }
8660d9f9647SVlad Buslov
8670d9f9647SVlad Buslov refcount_set(&e->refcnt, 1);
8680d9f9647SVlad Buslov init_completion(&e->res_ready);
869777bb800SVlad Buslov entry_created = true;
870777bb800SVlad Buslov INIT_LIST_HEAD(&e->route_list);
8710d9f9647SVlad Buslov
8720d9f9647SVlad Buslov tun_info = mlx5e_dup_tun_info(tun_info);
8730d9f9647SVlad Buslov if (!tun_info) {
8740d9f9647SVlad Buslov err = -ENOMEM;
8750d9f9647SVlad Buslov goto out_err_init;
8760d9f9647SVlad Buslov }
8770d9f9647SVlad Buslov e->tun_info = tun_info;
878c63741b4SMaor Dickman memcpy(&e->mpls_info, mpls_info, sizeof(*mpls_info));
8790d9f9647SVlad Buslov err = mlx5e_tc_tun_init_encap_attr(mirred_dev, priv, e, extack);
8800d9f9647SVlad Buslov if (err)
8810d9f9647SVlad Buslov goto out_err_init;
8820d9f9647SVlad Buslov
8830d9f9647SVlad Buslov INIT_LIST_HEAD(&e->flows);
8840d9f9647SVlad Buslov hash_add_rcu(esw->offloads.encap_tbl, &e->encap_hlist, hash_key);
8858914add2SVlad Buslov tbl_time_before = mlx5e_route_tbl_get_last_update(priv);
8860d9f9647SVlad Buslov
8870d9f9647SVlad Buslov if (family == AF_INET)
8880d9f9647SVlad Buslov err = mlx5e_tc_tun_create_header_ipv4(priv, mirred_dev, e);
8890d9f9647SVlad Buslov else if (family == AF_INET6)
8900d9f9647SVlad Buslov err = mlx5e_tc_tun_create_header_ipv6(priv, mirred_dev, e);
8910d9f9647SVlad Buslov
8920d9f9647SVlad Buslov complete_all(&e->res_ready);
8930d9f9647SVlad Buslov if (err) {
8940d9f9647SVlad Buslov e->compl_result = err;
8950d9f9647SVlad Buslov goto out_err;
8960d9f9647SVlad Buslov }
8970d9f9647SVlad Buslov e->compl_result = 1;
8980d9f9647SVlad Buslov
8990d9f9647SVlad Buslov attach_flow:
900c118ebc9SRoi Dayan err = mlx5e_attach_encap_route(priv, flow, attr, e, entry_created,
901c118ebc9SRoi Dayan tbl_time_before, out_index);
9020d9f9647SVlad Buslov if (err)
9030d9f9647SVlad Buslov goto out_err;
9040d9f9647SVlad Buslov
905100ad4e2SAriel Levkovich err = mlx5e_set_int_port_tunnel(priv, attr, e, out_index);
906100ad4e2SAriel Levkovich if (err == -EOPNOTSUPP) {
907100ad4e2SAriel Levkovich /* If device doesn't support int port offload,
908100ad4e2SAriel Levkovich * redirect to uplink vport.
909100ad4e2SAriel Levkovich */
910100ad4e2SAriel Levkovich mlx5_core_dbg(priv->mdev, "attaching int port as encap dev not supported, using uplink\n");
911100ad4e2SAriel Levkovich err = 0;
912100ad4e2SAriel Levkovich } else if (err) {
913100ad4e2SAriel Levkovich goto out_err;
914100ad4e2SAriel Levkovich }
915100ad4e2SAriel Levkovich
9160d9f9647SVlad Buslov flow->encaps[out_index].e = e;
9170d9f9647SVlad Buslov list_add(&flow->encaps[out_index].list, &e->flows);
9180d9f9647SVlad Buslov flow->encaps[out_index].index = out_index;
9190d9f9647SVlad Buslov *encap_dev = e->out_dev;
9200d9f9647SVlad Buslov if (e->flags & MLX5_ENCAP_ENTRY_VALID) {
9210d9f9647SVlad Buslov attr->esw_attr->dests[out_index].pkt_reformat = e->pkt_reformat;
9220d9f9647SVlad Buslov attr->esw_attr->dests[out_index].flags |= MLX5_ESW_DEST_ENCAP_VALID;
9230d9f9647SVlad Buslov } else {
924f3774220SChris Mi flow_flag_set(flow, SLOW);
9250d9f9647SVlad Buslov }
9260d9f9647SVlad Buslov
9270d9f9647SVlad Buslov return err;
9280d9f9647SVlad Buslov
9290d9f9647SVlad Buslov out_err:
9300d9f9647SVlad Buslov if (e)
93137c3b9faSChris Mi mlx5e_encap_put_locked(priv, e);
9320d9f9647SVlad Buslov return err;
9330d9f9647SVlad Buslov
9340d9f9647SVlad Buslov out_err_init:
9350d9f9647SVlad Buslov kfree(tun_info);
9360d9f9647SVlad Buslov kfree(e);
9370d9f9647SVlad Buslov return err;
9380d9f9647SVlad Buslov }
9390d9f9647SVlad Buslov
mlx5e_attach_decap(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow,struct netlink_ext_ack * extack)9400d9f9647SVlad Buslov int mlx5e_attach_decap(struct mlx5e_priv *priv,
9410d9f9647SVlad Buslov struct mlx5e_tc_flow *flow,
9420d9f9647SVlad Buslov struct netlink_ext_ack *extack)
9430d9f9647SVlad Buslov {
9440d9f9647SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
9450d9f9647SVlad Buslov struct mlx5_esw_flow_attr *attr = flow->attr->esw_attr;
9463f3f05abSYevgeny Kliteynik struct mlx5_pkt_reformat_params reformat_params;
9470d9f9647SVlad Buslov struct mlx5e_decap_entry *d;
9480d9f9647SVlad Buslov struct mlx5e_decap_key key;
9490d9f9647SVlad Buslov uintptr_t hash_key;
9500d9f9647SVlad Buslov int err = 0;
9510d9f9647SVlad Buslov
952697319b2SMaor Dickman if (sizeof(attr->eth) > MLX5_CAP_ESW(priv->mdev, max_encap_header_size)) {
9530d9f9647SVlad Buslov NL_SET_ERR_MSG_MOD(extack,
9540d9f9647SVlad Buslov "encap header larger than max supported");
9550d9f9647SVlad Buslov return -EOPNOTSUPP;
9560d9f9647SVlad Buslov }
9570d9f9647SVlad Buslov
958697319b2SMaor Dickman key.key = attr->eth;
9590d9f9647SVlad Buslov hash_key = hash_decap_info(&key);
9600d9f9647SVlad Buslov mutex_lock(&esw->offloads.decap_tbl_lock);
9610d9f9647SVlad Buslov d = mlx5e_decap_get(priv, &key, hash_key);
9620d9f9647SVlad Buslov if (d) {
9630d9f9647SVlad Buslov mutex_unlock(&esw->offloads.decap_tbl_lock);
9640d9f9647SVlad Buslov wait_for_completion(&d->res_ready);
9650d9f9647SVlad Buslov mutex_lock(&esw->offloads.decap_tbl_lock);
9660d9f9647SVlad Buslov if (d->compl_result) {
9670d9f9647SVlad Buslov err = -EREMOTEIO;
9680d9f9647SVlad Buslov goto out_free;
9690d9f9647SVlad Buslov }
9700d9f9647SVlad Buslov goto found;
9710d9f9647SVlad Buslov }
9720d9f9647SVlad Buslov
9730d9f9647SVlad Buslov d = kzalloc(sizeof(*d), GFP_KERNEL);
9740d9f9647SVlad Buslov if (!d) {
9750d9f9647SVlad Buslov err = -ENOMEM;
9760d9f9647SVlad Buslov goto out_err;
9770d9f9647SVlad Buslov }
9780d9f9647SVlad Buslov
9790d9f9647SVlad Buslov d->key = key;
9800d9f9647SVlad Buslov refcount_set(&d->refcnt, 1);
9810d9f9647SVlad Buslov init_completion(&d->res_ready);
9820d9f9647SVlad Buslov INIT_LIST_HEAD(&d->flows);
9830d9f9647SVlad Buslov hash_add_rcu(esw->offloads.decap_tbl, &d->hlist, hash_key);
9840d9f9647SVlad Buslov mutex_unlock(&esw->offloads.decap_tbl_lock);
9850d9f9647SVlad Buslov
9863f3f05abSYevgeny Kliteynik memset(&reformat_params, 0, sizeof(reformat_params));
9873f3f05abSYevgeny Kliteynik reformat_params.type = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
988697319b2SMaor Dickman reformat_params.size = sizeof(attr->eth);
989697319b2SMaor Dickman reformat_params.data = &attr->eth;
9900d9f9647SVlad Buslov d->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev,
9913f3f05abSYevgeny Kliteynik &reformat_params,
9920d9f9647SVlad Buslov MLX5_FLOW_NAMESPACE_FDB);
9930d9f9647SVlad Buslov if (IS_ERR(d->pkt_reformat)) {
9940d9f9647SVlad Buslov err = PTR_ERR(d->pkt_reformat);
9950d9f9647SVlad Buslov d->compl_result = err;
9960d9f9647SVlad Buslov }
9970d9f9647SVlad Buslov mutex_lock(&esw->offloads.decap_tbl_lock);
9980d9f9647SVlad Buslov complete_all(&d->res_ready);
9990d9f9647SVlad Buslov if (err)
10000d9f9647SVlad Buslov goto out_free;
10010d9f9647SVlad Buslov
10020d9f9647SVlad Buslov found:
10030d9f9647SVlad Buslov flow->decap_reformat = d;
10040d9f9647SVlad Buslov attr->decap_pkt_reformat = d->pkt_reformat;
10050d9f9647SVlad Buslov list_add(&flow->l3_to_l2_reformat, &d->flows);
10060d9f9647SVlad Buslov mutex_unlock(&esw->offloads.decap_tbl_lock);
10070d9f9647SVlad Buslov return 0;
10080d9f9647SVlad Buslov
10090d9f9647SVlad Buslov out_free:
10100d9f9647SVlad Buslov mutex_unlock(&esw->offloads.decap_tbl_lock);
10110d9f9647SVlad Buslov mlx5e_decap_put(priv, d);
10120d9f9647SVlad Buslov return err;
10130d9f9647SVlad Buslov
10140d9f9647SVlad Buslov out_err:
10150d9f9647SVlad Buslov mutex_unlock(&esw->offloads.decap_tbl_lock);
10160d9f9647SVlad Buslov return err;
10170d9f9647SVlad Buslov }
1018777bb800SVlad Buslov
mlx5e_tc_tun_encap_dests_set(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow,struct mlx5_flow_attr * attr,struct netlink_ext_ack * extack,bool * vf_tun)1019e2ab5aa1SChris Mi int mlx5e_tc_tun_encap_dests_set(struct mlx5e_priv *priv,
1020e2ab5aa1SChris Mi struct mlx5e_tc_flow *flow,
1021e2ab5aa1SChris Mi struct mlx5_flow_attr *attr,
1022e2ab5aa1SChris Mi struct netlink_ext_ack *extack,
1023e2ab5aa1SChris Mi bool *vf_tun)
1024e2ab5aa1SChris Mi {
1025e2ab5aa1SChris Mi struct mlx5e_tc_flow_parse_attr *parse_attr;
1026e2ab5aa1SChris Mi struct mlx5_esw_flow_attr *esw_attr;
1027e2ab5aa1SChris Mi struct net_device *encap_dev = NULL;
1028e2ab5aa1SChris Mi struct mlx5e_rep_priv *rpriv;
1029e2ab5aa1SChris Mi struct mlx5e_priv *out_priv;
103037c3b9faSChris Mi struct mlx5_eswitch *esw;
1031e2ab5aa1SChris Mi int out_index;
1032e2ab5aa1SChris Mi int err = 0;
1033e2ab5aa1SChris Mi
1034e2ab5aa1SChris Mi parse_attr = attr->parse_attr;
1035e2ab5aa1SChris Mi esw_attr = attr->esw_attr;
1036e2ab5aa1SChris Mi *vf_tun = false;
1037e2ab5aa1SChris Mi
103837c3b9faSChris Mi esw = priv->mdev->priv.eswitch;
103937c3b9faSChris Mi mutex_lock(&esw->offloads.encap_tbl_lock);
1040e2ab5aa1SChris Mi for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) {
1041e2ab5aa1SChris Mi struct net_device *out_dev;
1042e2ab5aa1SChris Mi int mirred_ifindex;
1043e2ab5aa1SChris Mi
1044e2ab5aa1SChris Mi if (!(esw_attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP))
1045e2ab5aa1SChris Mi continue;
1046e2ab5aa1SChris Mi
1047e2ab5aa1SChris Mi mirred_ifindex = parse_attr->mirred_ifindex[out_index];
1048e2ab5aa1SChris Mi out_dev = dev_get_by_index(dev_net(priv->netdev), mirred_ifindex);
1049e2ab5aa1SChris Mi if (!out_dev) {
1050e2ab5aa1SChris Mi NL_SET_ERR_MSG_MOD(extack, "Requested mirred device not found");
1051e2ab5aa1SChris Mi err = -ENODEV;
1052e2ab5aa1SChris Mi goto out;
1053e2ab5aa1SChris Mi }
1054e2ab5aa1SChris Mi err = mlx5e_attach_encap(priv, flow, attr, out_dev, out_index,
1055e2ab5aa1SChris Mi extack, &encap_dev);
1056e2ab5aa1SChris Mi dev_put(out_dev);
1057e2ab5aa1SChris Mi if (err)
1058e2ab5aa1SChris Mi goto out;
1059e2ab5aa1SChris Mi
1060e2ab5aa1SChris Mi if (esw_attr->dests[out_index].flags &
1061e2ab5aa1SChris Mi MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE &&
1062e2ab5aa1SChris Mi !esw_attr->dest_int_port)
1063e2ab5aa1SChris Mi *vf_tun = true;
1064e2ab5aa1SChris Mi
1065e2ab5aa1SChris Mi out_priv = netdev_priv(encap_dev);
1066e2ab5aa1SChris Mi rpriv = out_priv->ppriv;
1067*96c8c465SVlad Buslov esw_attr->dests[out_index].vport_valid = true;
1068*96c8c465SVlad Buslov esw_attr->dests[out_index].vport = rpriv->rep->vport;
1069e2ab5aa1SChris Mi esw_attr->dests[out_index].mdev = out_priv->mdev;
1070e2ab5aa1SChris Mi }
1071e2ab5aa1SChris Mi
1072e2ab5aa1SChris Mi if (*vf_tun && esw_attr->out_count > 1) {
1073e2ab5aa1SChris Mi NL_SET_ERR_MSG_MOD(extack, "VF tunnel encap with mirroring is not supported");
1074e2ab5aa1SChris Mi err = -EOPNOTSUPP;
1075e2ab5aa1SChris Mi goto out;
1076e2ab5aa1SChris Mi }
1077e2ab5aa1SChris Mi
1078e2ab5aa1SChris Mi out:
107937c3b9faSChris Mi mutex_unlock(&esw->offloads.encap_tbl_lock);
1080e2ab5aa1SChris Mi return err;
1081e2ab5aa1SChris Mi }
1082e2ab5aa1SChris Mi
mlx5e_tc_tun_encap_dests_unset(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow,struct mlx5_flow_attr * attr)1083e2ab5aa1SChris Mi void mlx5e_tc_tun_encap_dests_unset(struct mlx5e_priv *priv,
1084e2ab5aa1SChris Mi struct mlx5e_tc_flow *flow,
1085e2ab5aa1SChris Mi struct mlx5_flow_attr *attr)
1086e2ab5aa1SChris Mi {
1087e2ab5aa1SChris Mi struct mlx5_esw_flow_attr *esw_attr;
1088e2ab5aa1SChris Mi int out_index;
1089e2ab5aa1SChris Mi
1090e2ab5aa1SChris Mi if (!mlx5e_is_eswitch_flow(flow))
1091e2ab5aa1SChris Mi return;
1092e2ab5aa1SChris Mi
1093e2ab5aa1SChris Mi esw_attr = attr->esw_attr;
1094e2ab5aa1SChris Mi
1095e2ab5aa1SChris Mi for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) {
1096e2ab5aa1SChris Mi if (!(esw_attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP))
1097e2ab5aa1SChris Mi continue;
1098e2ab5aa1SChris Mi
1099e2ab5aa1SChris Mi mlx5e_detach_encap(flow->priv, flow, attr, out_index);
1100e2ab5aa1SChris Mi kfree(attr->parse_attr->tun_info[out_index]);
1101e2ab5aa1SChris Mi }
1102e2ab5aa1SChris Mi }
1103e2ab5aa1SChris Mi
cmp_route_info(struct mlx5e_route_key * a,struct mlx5e_route_key * b)1104777bb800SVlad Buslov static int cmp_route_info(struct mlx5e_route_key *a,
1105777bb800SVlad Buslov struct mlx5e_route_key *b)
1106777bb800SVlad Buslov {
1107777bb800SVlad Buslov if (a->ip_version == 4 && b->ip_version == 4)
1108777bb800SVlad Buslov return memcmp(&a->endpoint_ip.v4, &b->endpoint_ip.v4,
1109777bb800SVlad Buslov sizeof(a->endpoint_ip.v4));
1110777bb800SVlad Buslov else if (a->ip_version == 6 && b->ip_version == 6)
1111777bb800SVlad Buslov return memcmp(&a->endpoint_ip.v6, &b->endpoint_ip.v6,
1112777bb800SVlad Buslov sizeof(a->endpoint_ip.v6));
1113777bb800SVlad Buslov return 1;
1114777bb800SVlad Buslov }
1115777bb800SVlad Buslov
hash_route_info(struct mlx5e_route_key * key)1116777bb800SVlad Buslov static u32 hash_route_info(struct mlx5e_route_key *key)
1117777bb800SVlad Buslov {
1118777bb800SVlad Buslov if (key->ip_version == 4)
1119777bb800SVlad Buslov return jhash(&key->endpoint_ip.v4, sizeof(key->endpoint_ip.v4), 0);
1120777bb800SVlad Buslov return jhash(&key->endpoint_ip.v6, sizeof(key->endpoint_ip.v6), 0);
1121777bb800SVlad Buslov }
1122777bb800SVlad Buslov
mlx5e_route_dealloc(struct mlx5e_priv * priv,struct mlx5e_route_entry * r)11238914add2SVlad Buslov static void mlx5e_route_dealloc(struct mlx5e_priv *priv,
11248914add2SVlad Buslov struct mlx5e_route_entry *r)
11258914add2SVlad Buslov {
11268914add2SVlad Buslov WARN_ON(!list_empty(&r->decap_flows));
11278914add2SVlad Buslov WARN_ON(!list_empty(&r->encap_entries));
11288914add2SVlad Buslov
11298914add2SVlad Buslov kfree_rcu(r, rcu);
11308914add2SVlad Buslov }
11318914add2SVlad Buslov
mlx5e_route_put(struct mlx5e_priv * priv,struct mlx5e_route_entry * r)11328914add2SVlad Buslov static void mlx5e_route_put(struct mlx5e_priv *priv, struct mlx5e_route_entry *r)
1133777bb800SVlad Buslov {
1134777bb800SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
11358914add2SVlad Buslov
11368914add2SVlad Buslov if (!refcount_dec_and_mutex_lock(&r->refcnt, &esw->offloads.encap_tbl_lock))
11378914add2SVlad Buslov return;
11388914add2SVlad Buslov
11398914add2SVlad Buslov hash_del_rcu(&r->hlist);
11408914add2SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
11418914add2SVlad Buslov
11428914add2SVlad Buslov mlx5e_route_dealloc(priv, r);
11438914add2SVlad Buslov }
11448914add2SVlad Buslov
mlx5e_route_put_locked(struct mlx5e_priv * priv,struct mlx5e_route_entry * r)11458914add2SVlad Buslov static void mlx5e_route_put_locked(struct mlx5e_priv *priv, struct mlx5e_route_entry *r)
11468914add2SVlad Buslov {
11478914add2SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
11488914add2SVlad Buslov
11498914add2SVlad Buslov lockdep_assert_held(&esw->offloads.encap_tbl_lock);
11508914add2SVlad Buslov
11518914add2SVlad Buslov if (!refcount_dec_and_test(&r->refcnt))
11528914add2SVlad Buslov return;
11538914add2SVlad Buslov hash_del_rcu(&r->hlist);
11548914add2SVlad Buslov mlx5e_route_dealloc(priv, r);
11558914add2SVlad Buslov }
11568914add2SVlad Buslov
11578914add2SVlad Buslov static struct mlx5e_route_entry *
mlx5e_route_get(struct mlx5e_tc_tun_encap * encap,struct mlx5e_route_key * key,u32 hash_key)11588914add2SVlad Buslov mlx5e_route_get(struct mlx5e_tc_tun_encap *encap, struct mlx5e_route_key *key,
11598914add2SVlad Buslov u32 hash_key)
11608914add2SVlad Buslov {
1161777bb800SVlad Buslov struct mlx5e_route_key r_key;
1162777bb800SVlad Buslov struct mlx5e_route_entry *r;
1163777bb800SVlad Buslov
11648914add2SVlad Buslov hash_for_each_possible(encap->route_tbl, r, hlist, hash_key) {
1165777bb800SVlad Buslov r_key = r->key;
1166777bb800SVlad Buslov if (!cmp_route_info(&r_key, key) &&
1167777bb800SVlad Buslov refcount_inc_not_zero(&r->refcnt))
1168777bb800SVlad Buslov return r;
1169777bb800SVlad Buslov }
1170777bb800SVlad Buslov return NULL;
1171777bb800SVlad Buslov }
1172777bb800SVlad Buslov
1173777bb800SVlad Buslov static struct mlx5e_route_entry *
mlx5e_route_get_create(struct mlx5e_priv * priv,struct mlx5e_route_key * key,int tunnel_dev_index,unsigned long * route_tbl_change_time)1174777bb800SVlad Buslov mlx5e_route_get_create(struct mlx5e_priv *priv,
11758914add2SVlad Buslov struct mlx5e_route_key *key,
11768914add2SVlad Buslov int tunnel_dev_index,
11778914add2SVlad Buslov unsigned long *route_tbl_change_time)
1178777bb800SVlad Buslov {
1179777bb800SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
11808914add2SVlad Buslov struct mlx5_rep_uplink_priv *uplink_priv;
11818914add2SVlad Buslov struct mlx5e_rep_priv *uplink_rpriv;
11828914add2SVlad Buslov struct mlx5e_tc_tun_encap *encap;
1183777bb800SVlad Buslov struct mlx5e_route_entry *r;
1184777bb800SVlad Buslov u32 hash_key;
1185777bb800SVlad Buslov
11868914add2SVlad Buslov uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
11878914add2SVlad Buslov uplink_priv = &uplink_rpriv->uplink_priv;
11888914add2SVlad Buslov encap = uplink_priv->encap;
11898914add2SVlad Buslov
1190777bb800SVlad Buslov hash_key = hash_route_info(key);
11918914add2SVlad Buslov spin_lock_bh(&encap->route_lock);
11928914add2SVlad Buslov r = mlx5e_route_get(encap, key, hash_key);
11938914add2SVlad Buslov spin_unlock_bh(&encap->route_lock);
11948914add2SVlad Buslov if (r) {
11958914add2SVlad Buslov if (!mlx5e_route_entry_valid(r)) {
11968914add2SVlad Buslov mlx5e_route_put_locked(priv, r);
11978914add2SVlad Buslov return ERR_PTR(-EINVAL);
11988914add2SVlad Buslov }
1199777bb800SVlad Buslov return r;
12008914add2SVlad Buslov }
1201777bb800SVlad Buslov
1202777bb800SVlad Buslov r = kzalloc(sizeof(*r), GFP_KERNEL);
1203777bb800SVlad Buslov if (!r)
1204777bb800SVlad Buslov return ERR_PTR(-ENOMEM);
1205777bb800SVlad Buslov
1206777bb800SVlad Buslov r->key = *key;
12078914add2SVlad Buslov r->flags |= MLX5E_ROUTE_ENTRY_VALID;
12088914add2SVlad Buslov r->tunnel_dev_index = tunnel_dev_index;
1209777bb800SVlad Buslov refcount_set(&r->refcnt, 1);
1210777bb800SVlad Buslov INIT_LIST_HEAD(&r->decap_flows);
1211777bb800SVlad Buslov INIT_LIST_HEAD(&r->encap_entries);
12128914add2SVlad Buslov
12138914add2SVlad Buslov spin_lock_bh(&encap->route_lock);
12148914add2SVlad Buslov *route_tbl_change_time = encap->route_tbl_last_update;
12158914add2SVlad Buslov hash_add(encap->route_tbl, &r->hlist, hash_key);
12168914add2SVlad Buslov spin_unlock_bh(&encap->route_lock);
12178914add2SVlad Buslov
1218777bb800SVlad Buslov return r;
1219777bb800SVlad Buslov }
1220777bb800SVlad Buslov
12218914add2SVlad Buslov static struct mlx5e_route_entry *
mlx5e_route_lookup_for_update(struct mlx5e_tc_tun_encap * encap,struct mlx5e_route_key * key)12228914add2SVlad Buslov mlx5e_route_lookup_for_update(struct mlx5e_tc_tun_encap *encap, struct mlx5e_route_key *key)
12238914add2SVlad Buslov {
12248914add2SVlad Buslov u32 hash_key = hash_route_info(key);
12258914add2SVlad Buslov struct mlx5e_route_entry *r;
12268914add2SVlad Buslov
12278914add2SVlad Buslov spin_lock_bh(&encap->route_lock);
12288914add2SVlad Buslov encap->route_tbl_last_update = jiffies;
12298914add2SVlad Buslov r = mlx5e_route_get(encap, key, hash_key);
12308914add2SVlad Buslov spin_unlock_bh(&encap->route_lock);
12318914add2SVlad Buslov
12328914add2SVlad Buslov return r;
12338914add2SVlad Buslov }
12348914add2SVlad Buslov
12358914add2SVlad Buslov struct mlx5e_tc_fib_event_data {
12368914add2SVlad Buslov struct work_struct work;
12378914add2SVlad Buslov unsigned long event;
12388914add2SVlad Buslov struct mlx5e_route_entry *r;
12398914add2SVlad Buslov struct net_device *ul_dev;
12408914add2SVlad Buslov };
12418914add2SVlad Buslov
12428914add2SVlad Buslov static void mlx5e_tc_fib_event_work(struct work_struct *work);
12438914add2SVlad Buslov static struct mlx5e_tc_fib_event_data *
mlx5e_tc_init_fib_work(unsigned long event,struct net_device * ul_dev,gfp_t flags)12448914add2SVlad Buslov mlx5e_tc_init_fib_work(unsigned long event, struct net_device *ul_dev, gfp_t flags)
12458914add2SVlad Buslov {
12468914add2SVlad Buslov struct mlx5e_tc_fib_event_data *fib_work;
12478914add2SVlad Buslov
12488914add2SVlad Buslov fib_work = kzalloc(sizeof(*fib_work), flags);
12498914add2SVlad Buslov if (WARN_ON(!fib_work))
12508914add2SVlad Buslov return NULL;
12518914add2SVlad Buslov
12528914add2SVlad Buslov INIT_WORK(&fib_work->work, mlx5e_tc_fib_event_work);
12538914add2SVlad Buslov fib_work->event = event;
12548914add2SVlad Buslov fib_work->ul_dev = ul_dev;
12558914add2SVlad Buslov
12568914add2SVlad Buslov return fib_work;
12578914add2SVlad Buslov }
12588914add2SVlad Buslov
12598914add2SVlad Buslov static int
mlx5e_route_enqueue_update(struct mlx5e_priv * priv,struct mlx5e_route_entry * r,unsigned long event)12608914add2SVlad Buslov mlx5e_route_enqueue_update(struct mlx5e_priv *priv,
12618914add2SVlad Buslov struct mlx5e_route_entry *r,
12628914add2SVlad Buslov unsigned long event)
12638914add2SVlad Buslov {
12648914add2SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
12658914add2SVlad Buslov struct mlx5e_tc_fib_event_data *fib_work;
12668914add2SVlad Buslov struct mlx5e_rep_priv *uplink_rpriv;
12678914add2SVlad Buslov struct net_device *ul_dev;
12688914add2SVlad Buslov
12698914add2SVlad Buslov uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
12708914add2SVlad Buslov ul_dev = uplink_rpriv->netdev;
12718914add2SVlad Buslov
12728914add2SVlad Buslov fib_work = mlx5e_tc_init_fib_work(event, ul_dev, GFP_KERNEL);
12738914add2SVlad Buslov if (!fib_work)
12748914add2SVlad Buslov return -ENOMEM;
12758914add2SVlad Buslov
12768914add2SVlad Buslov dev_hold(ul_dev);
12778914add2SVlad Buslov refcount_inc(&r->refcnt);
12788914add2SVlad Buslov fib_work->r = r;
12798914add2SVlad Buslov queue_work(priv->wq, &fib_work->work);
12808914add2SVlad Buslov
12818914add2SVlad Buslov return 0;
12828914add2SVlad Buslov }
12838914add2SVlad Buslov
mlx5e_attach_decap_route(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow)1284777bb800SVlad Buslov int mlx5e_attach_decap_route(struct mlx5e_priv *priv,
1285777bb800SVlad Buslov struct mlx5e_tc_flow *flow)
1286777bb800SVlad Buslov {
1287777bb800SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
12888914add2SVlad Buslov unsigned long tbl_time_before, tbl_time_after;
1289777bb800SVlad Buslov struct mlx5e_tc_flow_parse_attr *parse_attr;
1290777bb800SVlad Buslov struct mlx5_flow_attr *attr = flow->attr;
1291777bb800SVlad Buslov struct mlx5_esw_flow_attr *esw_attr;
1292777bb800SVlad Buslov struct mlx5e_route_entry *r;
1293777bb800SVlad Buslov struct mlx5e_route_key key;
1294777bb800SVlad Buslov int err = 0;
1295777bb800SVlad Buslov
1296777bb800SVlad Buslov esw_attr = attr->esw_attr;
1297777bb800SVlad Buslov parse_attr = attr->parse_attr;
1298777bb800SVlad Buslov mutex_lock(&esw->offloads.encap_tbl_lock);
1299777bb800SVlad Buslov if (!esw_attr->rx_tun_attr)
1300777bb800SVlad Buslov goto out;
1301777bb800SVlad Buslov
13028914add2SVlad Buslov tbl_time_before = mlx5e_route_tbl_get_last_update(priv);
13038914add2SVlad Buslov tbl_time_after = tbl_time_before;
1304819c319cSChris Mi err = mlx5e_tc_tun_route_lookup(priv, &parse_attr->spec, attr, parse_attr->filter_dev);
1305777bb800SVlad Buslov if (err || !esw_attr->rx_tun_attr->decap_vport)
1306777bb800SVlad Buslov goto out;
1307777bb800SVlad Buslov
13081e74152eSRoi Dayan key.ip_version = attr->tun_ip_version;
1309777bb800SVlad Buslov if (key.ip_version == 4)
1310777bb800SVlad Buslov key.endpoint_ip.v4 = esw_attr->rx_tun_attr->dst_ip.v4;
1311777bb800SVlad Buslov else
1312777bb800SVlad Buslov key.endpoint_ip.v6 = esw_attr->rx_tun_attr->dst_ip.v6;
1313777bb800SVlad Buslov
13148914add2SVlad Buslov r = mlx5e_route_get_create(priv, &key, parse_attr->filter_dev->ifindex,
13158914add2SVlad Buslov &tbl_time_after);
1316777bb800SVlad Buslov if (IS_ERR(r)) {
1317777bb800SVlad Buslov err = PTR_ERR(r);
1318777bb800SVlad Buslov goto out;
1319777bb800SVlad Buslov }
13208914add2SVlad Buslov /* Routing changed concurrently. FIB event handler might have missed new
13218914add2SVlad Buslov * entry, schedule update.
13228914add2SVlad Buslov */
13238914add2SVlad Buslov if (tbl_time_before != tbl_time_after) {
13248914add2SVlad Buslov err = mlx5e_route_enqueue_update(priv, r, FIB_EVENT_ENTRY_REPLACE);
13258914add2SVlad Buslov if (err) {
13268914add2SVlad Buslov mlx5e_route_put_locked(priv, r);
13278914add2SVlad Buslov goto out;
13288914add2SVlad Buslov }
13298914add2SVlad Buslov }
1330777bb800SVlad Buslov
1331777bb800SVlad Buslov flow->decap_route = r;
1332777bb800SVlad Buslov list_add(&flow->decap_routes, &r->decap_flows);
1333777bb800SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
1334777bb800SVlad Buslov return 0;
1335777bb800SVlad Buslov
1336777bb800SVlad Buslov out:
1337777bb800SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
1338777bb800SVlad Buslov return err;
1339777bb800SVlad Buslov }
1340777bb800SVlad Buslov
mlx5e_attach_encap_route(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow,struct mlx5_flow_attr * attr,struct mlx5e_encap_entry * e,bool new_encap_entry,unsigned long tbl_time_before,int out_index)1341777bb800SVlad Buslov static int mlx5e_attach_encap_route(struct mlx5e_priv *priv,
1342777bb800SVlad Buslov struct mlx5e_tc_flow *flow,
1343c118ebc9SRoi Dayan struct mlx5_flow_attr *attr,
1344777bb800SVlad Buslov struct mlx5e_encap_entry *e,
1345777bb800SVlad Buslov bool new_encap_entry,
13468914add2SVlad Buslov unsigned long tbl_time_before,
1347777bb800SVlad Buslov int out_index)
1348777bb800SVlad Buslov {
1349777bb800SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
13508914add2SVlad Buslov unsigned long tbl_time_after = tbl_time_before;
1351777bb800SVlad Buslov struct mlx5e_tc_flow_parse_attr *parse_attr;
1352777bb800SVlad Buslov const struct ip_tunnel_info *tun_info;
1353777bb800SVlad Buslov struct mlx5_esw_flow_attr *esw_attr;
1354777bb800SVlad Buslov struct mlx5e_route_entry *r;
1355777bb800SVlad Buslov struct mlx5e_route_key key;
1356777bb800SVlad Buslov unsigned short family;
1357777bb800SVlad Buslov int err = 0;
1358777bb800SVlad Buslov
1359777bb800SVlad Buslov esw_attr = attr->esw_attr;
1360777bb800SVlad Buslov parse_attr = attr->parse_attr;
1361777bb800SVlad Buslov tun_info = parse_attr->tun_info[out_index];
1362777bb800SVlad Buslov family = ip_tunnel_info_af(tun_info);
1363777bb800SVlad Buslov
1364777bb800SVlad Buslov if (family == AF_INET) {
1365777bb800SVlad Buslov key.endpoint_ip.v4 = tun_info->key.u.ipv4.src;
1366777bb800SVlad Buslov key.ip_version = 4;
1367777bb800SVlad Buslov } else if (family == AF_INET6) {
1368777bb800SVlad Buslov key.endpoint_ip.v6 = tun_info->key.u.ipv6.src;
1369777bb800SVlad Buslov key.ip_version = 6;
1370777bb800SVlad Buslov }
1371777bb800SVlad Buslov
1372777bb800SVlad Buslov err = mlx5e_set_vf_tunnel(esw, attr, &parse_attr->mod_hdr_acts, e->out_dev,
1373777bb800SVlad Buslov e->route_dev_ifindex, out_index);
1374777bb800SVlad Buslov if (err || !(esw_attr->dests[out_index].flags &
1375777bb800SVlad Buslov MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE))
1376777bb800SVlad Buslov return err;
1377777bb800SVlad Buslov
13788914add2SVlad Buslov r = mlx5e_route_get_create(priv, &key, parse_attr->mirred_ifindex[out_index],
13798914add2SVlad Buslov &tbl_time_after);
1380777bb800SVlad Buslov if (IS_ERR(r))
1381777bb800SVlad Buslov return PTR_ERR(r);
13828914add2SVlad Buslov /* Routing changed concurrently. FIB event handler might have missed new
13838914add2SVlad Buslov * entry, schedule update.
13848914add2SVlad Buslov */
13858914add2SVlad Buslov if (tbl_time_before != tbl_time_after) {
13868914add2SVlad Buslov err = mlx5e_route_enqueue_update(priv, r, FIB_EVENT_ENTRY_REPLACE);
13878914add2SVlad Buslov if (err) {
13888914add2SVlad Buslov mlx5e_route_put_locked(priv, r);
13898914add2SVlad Buslov return err;
13908914add2SVlad Buslov }
13918914add2SVlad Buslov }
1392777bb800SVlad Buslov
1393777bb800SVlad Buslov flow->encap_routes[out_index].r = r;
1394777bb800SVlad Buslov if (new_encap_entry)
1395777bb800SVlad Buslov list_add(&e->route_list, &r->encap_entries);
1396777bb800SVlad Buslov flow->encap_routes[out_index].index = out_index;
1397777bb800SVlad Buslov return 0;
1398777bb800SVlad Buslov }
1399777bb800SVlad Buslov
mlx5e_detach_decap_route(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow)1400777bb800SVlad Buslov void mlx5e_detach_decap_route(struct mlx5e_priv *priv,
1401777bb800SVlad Buslov struct mlx5e_tc_flow *flow)
1402777bb800SVlad Buslov {
1403777bb800SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1404777bb800SVlad Buslov struct mlx5e_route_entry *r = flow->decap_route;
1405777bb800SVlad Buslov
1406777bb800SVlad Buslov if (!r)
1407777bb800SVlad Buslov return;
1408777bb800SVlad Buslov
1409777bb800SVlad Buslov mutex_lock(&esw->offloads.encap_tbl_lock);
1410777bb800SVlad Buslov list_del(&flow->decap_routes);
1411777bb800SVlad Buslov flow->decap_route = NULL;
1412777bb800SVlad Buslov
1413777bb800SVlad Buslov if (!refcount_dec_and_test(&r->refcnt)) {
1414777bb800SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
1415777bb800SVlad Buslov return;
1416777bb800SVlad Buslov }
1417777bb800SVlad Buslov hash_del_rcu(&r->hlist);
1418777bb800SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
1419777bb800SVlad Buslov
1420777bb800SVlad Buslov mlx5e_route_dealloc(priv, r);
1421777bb800SVlad Buslov }
1422777bb800SVlad Buslov
mlx5e_detach_encap_route(struct mlx5e_priv * priv,struct mlx5e_tc_flow * flow,int out_index)1423777bb800SVlad Buslov static void mlx5e_detach_encap_route(struct mlx5e_priv *priv,
1424777bb800SVlad Buslov struct mlx5e_tc_flow *flow,
1425777bb800SVlad Buslov int out_index)
1426777bb800SVlad Buslov {
1427777bb800SVlad Buslov struct mlx5e_route_entry *r = flow->encap_routes[out_index].r;
1428777bb800SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1429777bb800SVlad Buslov struct mlx5e_encap_entry *e, *tmp;
1430777bb800SVlad Buslov
1431777bb800SVlad Buslov if (!r)
1432777bb800SVlad Buslov return;
1433777bb800SVlad Buslov
1434777bb800SVlad Buslov mutex_lock(&esw->offloads.encap_tbl_lock);
1435777bb800SVlad Buslov flow->encap_routes[out_index].r = NULL;
1436777bb800SVlad Buslov
1437777bb800SVlad Buslov if (!refcount_dec_and_test(&r->refcnt)) {
1438777bb800SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
1439777bb800SVlad Buslov return;
1440777bb800SVlad Buslov }
1441777bb800SVlad Buslov list_for_each_entry_safe(e, tmp, &r->encap_entries, route_list)
1442777bb800SVlad Buslov list_del_init(&e->route_list);
1443777bb800SVlad Buslov hash_del_rcu(&r->hlist);
1444777bb800SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
1445777bb800SVlad Buslov
1446777bb800SVlad Buslov mlx5e_route_dealloc(priv, r);
1447777bb800SVlad Buslov }
1448777bb800SVlad Buslov
mlx5e_invalidate_encap(struct mlx5e_priv * priv,struct mlx5e_encap_entry * e,struct list_head * encap_flows)14498914add2SVlad Buslov static void mlx5e_invalidate_encap(struct mlx5e_priv *priv,
14508914add2SVlad Buslov struct mlx5e_encap_entry *e,
14518914add2SVlad Buslov struct list_head *encap_flows)
14528914add2SVlad Buslov {
14538914add2SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
14548914add2SVlad Buslov struct mlx5e_tc_flow *flow;
14558914add2SVlad Buslov
14568914add2SVlad Buslov list_for_each_entry(flow, encap_flows, tmp_list) {
14578914add2SVlad Buslov struct mlx5_esw_flow_attr *esw_attr;
1458be071cdbSVlad Buslov struct mlx5_flow_attr *attr;
14598914add2SVlad Buslov
14608914add2SVlad Buslov if (!mlx5e_is_offloaded_flow(flow))
14618914add2SVlad Buslov continue;
1462be071cdbSVlad Buslov
1463be071cdbSVlad Buslov attr = mlx5e_tc_get_encap_attr(flow);
14648914add2SVlad Buslov esw_attr = attr->esw_attr;
14658914add2SVlad Buslov
14666b5926ebSChris Mi if (flow_flag_test(flow, SLOW)) {
14678914add2SVlad Buslov mlx5e_tc_unoffload_from_slow_path(esw, flow);
14686b5926ebSChris Mi } else {
14698914add2SVlad Buslov mlx5e_tc_unoffload_fdb_rules(esw, flow, flow->attr);
14706b5926ebSChris Mi mlx5e_tc_unoffload_flow_post_acts(flow);
14716b5926ebSChris Mi }
1472ef78b8d5SRoi Dayan
1473ef78b8d5SRoi Dayan mlx5e_tc_detach_mod_hdr(priv, flow, attr);
14748914add2SVlad Buslov attr->modify_hdr = NULL;
14758914add2SVlad Buslov
14768914add2SVlad Buslov esw_attr->dests[flow->tmp_entry_index].flags &=
14778914add2SVlad Buslov ~MLX5_ESW_DEST_ENCAP_VALID;
14788914add2SVlad Buslov esw_attr->dests[flow->tmp_entry_index].pkt_reformat = NULL;
14798914add2SVlad Buslov }
14808914add2SVlad Buslov
14818914add2SVlad Buslov e->flags |= MLX5_ENCAP_ENTRY_NO_ROUTE;
14828914add2SVlad Buslov if (e->flags & MLX5_ENCAP_ENTRY_VALID) {
14838914add2SVlad Buslov e->flags &= ~MLX5_ENCAP_ENTRY_VALID;
14848914add2SVlad Buslov mlx5_packet_reformat_dealloc(priv->mdev, e->pkt_reformat);
14858914add2SVlad Buslov e->pkt_reformat = NULL;
14868914add2SVlad Buslov }
14878914add2SVlad Buslov }
14888914add2SVlad Buslov
mlx5e_reoffload_encap(struct mlx5e_priv * priv,struct net_device * tunnel_dev,struct mlx5e_encap_entry * e,struct list_head * encap_flows)14898914add2SVlad Buslov static void mlx5e_reoffload_encap(struct mlx5e_priv *priv,
14908914add2SVlad Buslov struct net_device *tunnel_dev,
14918914add2SVlad Buslov struct mlx5e_encap_entry *e,
14928914add2SVlad Buslov struct list_head *encap_flows)
14938914add2SVlad Buslov {
14948914add2SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
14958914add2SVlad Buslov struct mlx5e_tc_flow *flow;
14968914add2SVlad Buslov int err;
14978914add2SVlad Buslov
14988914add2SVlad Buslov err = ip_tunnel_info_af(e->tun_info) == AF_INET ?
14998914add2SVlad Buslov mlx5e_tc_tun_update_header_ipv4(priv, tunnel_dev, e) :
15008914add2SVlad Buslov mlx5e_tc_tun_update_header_ipv6(priv, tunnel_dev, e);
15018914add2SVlad Buslov if (err)
15028914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update encap header, %d", err);
15038914add2SVlad Buslov e->flags &= ~MLX5_ENCAP_ENTRY_NO_ROUTE;
15048914add2SVlad Buslov
15058914add2SVlad Buslov list_for_each_entry(flow, encap_flows, tmp_list) {
15068914add2SVlad Buslov struct mlx5e_tc_flow_parse_attr *parse_attr;
15078914add2SVlad Buslov struct mlx5_esw_flow_attr *esw_attr;
15088914add2SVlad Buslov struct mlx5_flow_handle *rule;
15098300f225SRoi Dayan struct mlx5_flow_attr *attr;
15108914add2SVlad Buslov struct mlx5_flow_spec *spec;
15118914add2SVlad Buslov
15128914add2SVlad Buslov if (flow_flag_test(flow, FAILED))
15138914add2SVlad Buslov continue;
15148914add2SVlad Buslov
15158300f225SRoi Dayan spec = &flow->attr->parse_attr->spec;
15168300f225SRoi Dayan
15178300f225SRoi Dayan attr = mlx5e_tc_get_encap_attr(flow);
15188914add2SVlad Buslov esw_attr = attr->esw_attr;
15198914add2SVlad Buslov parse_attr = attr->parse_attr;
15208914add2SVlad Buslov
15218914add2SVlad Buslov err = mlx5e_update_vf_tunnel(esw, esw_attr, &parse_attr->mod_hdr_acts,
15228914add2SVlad Buslov e->out_dev, e->route_dev_ifindex,
15238914add2SVlad Buslov flow->tmp_entry_index);
15248914add2SVlad Buslov if (err) {
15258914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update VF tunnel err=%d", err);
15268914add2SVlad Buslov continue;
15278914add2SVlad Buslov }
15288914add2SVlad Buslov
1529ef78b8d5SRoi Dayan err = mlx5e_tc_attach_mod_hdr(priv, flow, attr);
15308914add2SVlad Buslov if (err) {
15318914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update flow mod_hdr err=%d",
15328914add2SVlad Buslov err);
15338914add2SVlad Buslov continue;
15348914add2SVlad Buslov }
15358914add2SVlad Buslov
15368914add2SVlad Buslov if (e->flags & MLX5_ENCAP_ENTRY_VALID) {
15378914add2SVlad Buslov esw_attr->dests[flow->tmp_entry_index].pkt_reformat = e->pkt_reformat;
15388914add2SVlad Buslov esw_attr->dests[flow->tmp_entry_index].flags |= MLX5_ESW_DEST_ENCAP_VALID;
15398914add2SVlad Buslov if (!mlx5e_tc_flow_all_encaps_valid(esw_attr))
15408914add2SVlad Buslov goto offload_to_slow_path;
15418300f225SRoi Dayan
15428300f225SRoi Dayan err = mlx5e_tc_offload_flow_post_acts(flow);
15438300f225SRoi Dayan if (err) {
15448300f225SRoi Dayan mlx5_core_warn(priv->mdev, "Failed to update flow post acts, %d\n",
15458300f225SRoi Dayan err);
15468300f225SRoi Dayan goto offload_to_slow_path;
15478300f225SRoi Dayan }
15488300f225SRoi Dayan
15498914add2SVlad Buslov /* update from slow path rule to encap rule */
15508300f225SRoi Dayan rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, flow->attr);
15518914add2SVlad Buslov if (IS_ERR(rule)) {
15528300f225SRoi Dayan mlx5e_tc_unoffload_flow_post_acts(flow);
15538914add2SVlad Buslov err = PTR_ERR(rule);
15548914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update cached encapsulation flow, %d\n",
15558914add2SVlad Buslov err);
15568914add2SVlad Buslov } else {
15578914add2SVlad Buslov flow->rule[0] = rule;
15588914add2SVlad Buslov }
15598914add2SVlad Buslov } else {
15608914add2SVlad Buslov offload_to_slow_path:
15618914add2SVlad Buslov rule = mlx5e_tc_offload_to_slow_path(esw, flow, spec);
15628914add2SVlad Buslov /* mark the flow's encap dest as non-valid */
15638914add2SVlad Buslov esw_attr->dests[flow->tmp_entry_index].flags &=
15648914add2SVlad Buslov ~MLX5_ESW_DEST_ENCAP_VALID;
15658914add2SVlad Buslov
15668914add2SVlad Buslov if (IS_ERR(rule)) {
15678914add2SVlad Buslov err = PTR_ERR(rule);
15688914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update slow path (encap) flow, %d\n",
15698914add2SVlad Buslov err);
15708914add2SVlad Buslov } else {
15718914add2SVlad Buslov flow->rule[0] = rule;
15728914add2SVlad Buslov }
15738914add2SVlad Buslov }
15748914add2SVlad Buslov flow_flag_set(flow, OFFLOADED);
15758914add2SVlad Buslov }
15768914add2SVlad Buslov }
15778914add2SVlad Buslov
mlx5e_update_route_encaps(struct mlx5e_priv * priv,struct mlx5e_route_entry * r,struct list_head * flow_list,bool replace)15788914add2SVlad Buslov static int mlx5e_update_route_encaps(struct mlx5e_priv *priv,
15798914add2SVlad Buslov struct mlx5e_route_entry *r,
15808914add2SVlad Buslov struct list_head *flow_list,
15818914add2SVlad Buslov bool replace)
15828914add2SVlad Buslov {
15838914add2SVlad Buslov struct net_device *tunnel_dev;
15848914add2SVlad Buslov struct mlx5e_encap_entry *e;
15858914add2SVlad Buslov
15868914add2SVlad Buslov tunnel_dev = __dev_get_by_index(dev_net(priv->netdev), r->tunnel_dev_index);
15878914add2SVlad Buslov if (!tunnel_dev)
15888914add2SVlad Buslov return -ENODEV;
15898914add2SVlad Buslov
15908914add2SVlad Buslov list_for_each_entry(e, &r->encap_entries, route_list) {
15918914add2SVlad Buslov LIST_HEAD(encap_flows);
15928914add2SVlad Buslov
15938914add2SVlad Buslov mlx5e_take_all_encap_flows(e, &encap_flows);
15948914add2SVlad Buslov if (list_empty(&encap_flows))
15958914add2SVlad Buslov continue;
15968914add2SVlad Buslov
15978914add2SVlad Buslov if (mlx5e_route_entry_valid(r))
15988914add2SVlad Buslov mlx5e_invalidate_encap(priv, e, &encap_flows);
15998914add2SVlad Buslov
16008914add2SVlad Buslov if (!replace) {
16018914add2SVlad Buslov list_splice(&encap_flows, flow_list);
16028914add2SVlad Buslov continue;
16038914add2SVlad Buslov }
16048914add2SVlad Buslov
16058914add2SVlad Buslov mlx5e_reoffload_encap(priv, tunnel_dev, e, &encap_flows);
16068914add2SVlad Buslov list_splice(&encap_flows, flow_list);
16078914add2SVlad Buslov }
16088914add2SVlad Buslov
16098914add2SVlad Buslov return 0;
16108914add2SVlad Buslov }
16118914add2SVlad Buslov
mlx5e_unoffload_flow_list(struct mlx5e_priv * priv,struct list_head * flow_list)16128914add2SVlad Buslov static void mlx5e_unoffload_flow_list(struct mlx5e_priv *priv,
16138914add2SVlad Buslov struct list_head *flow_list)
16148914add2SVlad Buslov {
16158914add2SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
16168914add2SVlad Buslov struct mlx5e_tc_flow *flow;
16178914add2SVlad Buslov
16188914add2SVlad Buslov list_for_each_entry(flow, flow_list, tmp_list)
16198914add2SVlad Buslov if (mlx5e_is_offloaded_flow(flow))
16208914add2SVlad Buslov mlx5e_tc_unoffload_fdb_rules(esw, flow, flow->attr);
16218914add2SVlad Buslov }
16228914add2SVlad Buslov
mlx5e_reoffload_decap(struct mlx5e_priv * priv,struct list_head * decap_flows)16238914add2SVlad Buslov static void mlx5e_reoffload_decap(struct mlx5e_priv *priv,
16248914add2SVlad Buslov struct list_head *decap_flows)
16258914add2SVlad Buslov {
16268914add2SVlad Buslov struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
16278914add2SVlad Buslov struct mlx5e_tc_flow *flow;
16288914add2SVlad Buslov
16298914add2SVlad Buslov list_for_each_entry(flow, decap_flows, tmp_list) {
16308914add2SVlad Buslov struct mlx5e_tc_flow_parse_attr *parse_attr;
16318914add2SVlad Buslov struct mlx5_flow_attr *attr = flow->attr;
16328914add2SVlad Buslov struct mlx5_flow_handle *rule;
16338914add2SVlad Buslov struct mlx5_flow_spec *spec;
16348914add2SVlad Buslov int err;
16358914add2SVlad Buslov
16368914add2SVlad Buslov if (flow_flag_test(flow, FAILED))
16378914add2SVlad Buslov continue;
16388914add2SVlad Buslov
16398914add2SVlad Buslov parse_attr = attr->parse_attr;
16408914add2SVlad Buslov spec = &parse_attr->spec;
1641819c319cSChris Mi err = mlx5e_tc_tun_route_lookup(priv, spec, attr, parse_attr->filter_dev);
16428914add2SVlad Buslov if (err) {
16438914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to lookup route for flow, %d\n",
16448914add2SVlad Buslov err);
16458914add2SVlad Buslov continue;
16468914add2SVlad Buslov }
16478914add2SVlad Buslov
16488914add2SVlad Buslov rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, attr);
16498914add2SVlad Buslov if (IS_ERR(rule)) {
16508914add2SVlad Buslov err = PTR_ERR(rule);
16518914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update cached decap flow, %d\n",
16528914add2SVlad Buslov err);
16538914add2SVlad Buslov } else {
16548914add2SVlad Buslov flow->rule[0] = rule;
16558914add2SVlad Buslov flow_flag_set(flow, OFFLOADED);
16568914add2SVlad Buslov }
16578914add2SVlad Buslov }
16588914add2SVlad Buslov }
16598914add2SVlad Buslov
mlx5e_update_route_decap_flows(struct mlx5e_priv * priv,struct mlx5e_route_entry * r,struct list_head * flow_list,bool replace)16608914add2SVlad Buslov static int mlx5e_update_route_decap_flows(struct mlx5e_priv *priv,
16618914add2SVlad Buslov struct mlx5e_route_entry *r,
16628914add2SVlad Buslov struct list_head *flow_list,
16638914add2SVlad Buslov bool replace)
16648914add2SVlad Buslov {
16658914add2SVlad Buslov struct net_device *tunnel_dev;
16668914add2SVlad Buslov LIST_HEAD(decap_flows);
16678914add2SVlad Buslov
16688914add2SVlad Buslov tunnel_dev = __dev_get_by_index(dev_net(priv->netdev), r->tunnel_dev_index);
16698914add2SVlad Buslov if (!tunnel_dev)
16708914add2SVlad Buslov return -ENODEV;
16718914add2SVlad Buslov
16728914add2SVlad Buslov mlx5e_take_all_route_decap_flows(r, &decap_flows);
16738914add2SVlad Buslov if (mlx5e_route_entry_valid(r))
16748914add2SVlad Buslov mlx5e_unoffload_flow_list(priv, &decap_flows);
16758914add2SVlad Buslov if (replace)
16768914add2SVlad Buslov mlx5e_reoffload_decap(priv, &decap_flows);
16778914add2SVlad Buslov
16788914add2SVlad Buslov list_splice(&decap_flows, flow_list);
16798914add2SVlad Buslov
16808914add2SVlad Buslov return 0;
16818914add2SVlad Buslov }
16828914add2SVlad Buslov
mlx5e_tc_fib_event_work(struct work_struct * work)16838914add2SVlad Buslov static void mlx5e_tc_fib_event_work(struct work_struct *work)
16848914add2SVlad Buslov {
16858914add2SVlad Buslov struct mlx5e_tc_fib_event_data *event_data =
16868914add2SVlad Buslov container_of(work, struct mlx5e_tc_fib_event_data, work);
16878914add2SVlad Buslov struct net_device *ul_dev = event_data->ul_dev;
16888914add2SVlad Buslov struct mlx5e_priv *priv = netdev_priv(ul_dev);
16898914add2SVlad Buslov struct mlx5e_route_entry *r = event_data->r;
16908914add2SVlad Buslov struct mlx5_eswitch *esw;
16918914add2SVlad Buslov LIST_HEAD(flow_list);
16928914add2SVlad Buslov bool replace;
16938914add2SVlad Buslov int err;
16948914add2SVlad Buslov
16958914add2SVlad Buslov /* sync with concurrent neigh updates */
16968914add2SVlad Buslov rtnl_lock();
16978914add2SVlad Buslov esw = priv->mdev->priv.eswitch;
16988914add2SVlad Buslov mutex_lock(&esw->offloads.encap_tbl_lock);
16998914add2SVlad Buslov replace = event_data->event == FIB_EVENT_ENTRY_REPLACE;
17008914add2SVlad Buslov
17018914add2SVlad Buslov if (!mlx5e_route_entry_valid(r) && !replace)
17028914add2SVlad Buslov goto out;
17038914add2SVlad Buslov
17048914add2SVlad Buslov err = mlx5e_update_route_encaps(priv, r, &flow_list, replace);
17058914add2SVlad Buslov if (err)
17068914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update route encaps, %d\n",
17078914add2SVlad Buslov err);
17088914add2SVlad Buslov
17098914add2SVlad Buslov err = mlx5e_update_route_decap_flows(priv, r, &flow_list, replace);
17108914add2SVlad Buslov if (err)
17118914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to update route decap flows, %d\n",
17128914add2SVlad Buslov err);
17138914add2SVlad Buslov
17148914add2SVlad Buslov if (replace)
17158914add2SVlad Buslov r->flags |= MLX5E_ROUTE_ENTRY_VALID;
17168914add2SVlad Buslov out:
17178914add2SVlad Buslov mutex_unlock(&esw->offloads.encap_tbl_lock);
17188914add2SVlad Buslov rtnl_unlock();
17198914add2SVlad Buslov
17208914add2SVlad Buslov mlx5e_put_flow_list(priv, &flow_list);
17218914add2SVlad Buslov mlx5e_route_put(priv, event_data->r);
17228914add2SVlad Buslov dev_put(event_data->ul_dev);
17238914add2SVlad Buslov kfree(event_data);
17248914add2SVlad Buslov }
17258914add2SVlad Buslov
17268914add2SVlad Buslov static struct mlx5e_tc_fib_event_data *
mlx5e_init_fib_work_ipv4(struct mlx5e_priv * priv,struct net_device * ul_dev,struct mlx5e_tc_tun_encap * encap,unsigned long event,struct fib_notifier_info * info)17278914add2SVlad Buslov mlx5e_init_fib_work_ipv4(struct mlx5e_priv *priv,
17288914add2SVlad Buslov struct net_device *ul_dev,
17298914add2SVlad Buslov struct mlx5e_tc_tun_encap *encap,
17308914add2SVlad Buslov unsigned long event,
17318914add2SVlad Buslov struct fib_notifier_info *info)
17328914add2SVlad Buslov {
17338914add2SVlad Buslov struct fib_entry_notifier_info *fen_info;
17348914add2SVlad Buslov struct mlx5e_tc_fib_event_data *fib_work;
17358914add2SVlad Buslov struct mlx5e_route_entry *r;
17368914add2SVlad Buslov struct mlx5e_route_key key;
17378914add2SVlad Buslov struct net_device *fib_dev;
17388914add2SVlad Buslov
17398914add2SVlad Buslov fen_info = container_of(info, struct fib_entry_notifier_info, info);
1740885751ebSMaor Dickman if (fen_info->fi->nh)
1741885751ebSMaor Dickman return NULL;
17428914add2SVlad Buslov fib_dev = fib_info_nh(fen_info->fi, 0)->fib_nh_dev;
1743eb96cc15SRoi Dayan if (!fib_dev || fib_dev->netdev_ops != &mlx5e_netdev_ops ||
17448914add2SVlad Buslov fen_info->dst_len != 32)
17458914add2SVlad Buslov return NULL;
17468914add2SVlad Buslov
17478914add2SVlad Buslov fib_work = mlx5e_tc_init_fib_work(event, ul_dev, GFP_ATOMIC);
17488914add2SVlad Buslov if (!fib_work)
17498914add2SVlad Buslov return ERR_PTR(-ENOMEM);
17508914add2SVlad Buslov
17518914add2SVlad Buslov key.endpoint_ip.v4 = htonl(fen_info->dst);
17528914add2SVlad Buslov key.ip_version = 4;
17538914add2SVlad Buslov
17548914add2SVlad Buslov /* Can't fail after this point because releasing reference to r
17558914add2SVlad Buslov * requires obtaining sleeping mutex which we can't do in atomic
17568914add2SVlad Buslov * context.
17578914add2SVlad Buslov */
17588914add2SVlad Buslov r = mlx5e_route_lookup_for_update(encap, &key);
17598914add2SVlad Buslov if (!r)
17608914add2SVlad Buslov goto out;
17618914add2SVlad Buslov fib_work->r = r;
17628914add2SVlad Buslov dev_hold(ul_dev);
17638914add2SVlad Buslov
17648914add2SVlad Buslov return fib_work;
17658914add2SVlad Buslov
17668914add2SVlad Buslov out:
17678914add2SVlad Buslov kfree(fib_work);
17688914add2SVlad Buslov return NULL;
17698914add2SVlad Buslov }
17708914add2SVlad Buslov
17718914add2SVlad Buslov static struct mlx5e_tc_fib_event_data *
mlx5e_init_fib_work_ipv6(struct mlx5e_priv * priv,struct net_device * ul_dev,struct mlx5e_tc_tun_encap * encap,unsigned long event,struct fib_notifier_info * info)17728914add2SVlad Buslov mlx5e_init_fib_work_ipv6(struct mlx5e_priv *priv,
17738914add2SVlad Buslov struct net_device *ul_dev,
17748914add2SVlad Buslov struct mlx5e_tc_tun_encap *encap,
17758914add2SVlad Buslov unsigned long event,
17768914add2SVlad Buslov struct fib_notifier_info *info)
17778914add2SVlad Buslov {
17788914add2SVlad Buslov struct fib6_entry_notifier_info *fen_info;
17798914add2SVlad Buslov struct mlx5e_tc_fib_event_data *fib_work;
17808914add2SVlad Buslov struct mlx5e_route_entry *r;
17818914add2SVlad Buslov struct mlx5e_route_key key;
17828914add2SVlad Buslov struct net_device *fib_dev;
17838914add2SVlad Buslov
17848914add2SVlad Buslov fen_info = container_of(info, struct fib6_entry_notifier_info, info);
17858914add2SVlad Buslov fib_dev = fib6_info_nh_dev(fen_info->rt);
17868914add2SVlad Buslov if (fib_dev->netdev_ops != &mlx5e_netdev_ops ||
17878914add2SVlad Buslov fen_info->rt->fib6_dst.plen != 128)
17888914add2SVlad Buslov return NULL;
17898914add2SVlad Buslov
17908914add2SVlad Buslov fib_work = mlx5e_tc_init_fib_work(event, ul_dev, GFP_ATOMIC);
17918914add2SVlad Buslov if (!fib_work)
17928914add2SVlad Buslov return ERR_PTR(-ENOMEM);
17938914add2SVlad Buslov
17948914add2SVlad Buslov memcpy(&key.endpoint_ip.v6, &fen_info->rt->fib6_dst.addr,
17958914add2SVlad Buslov sizeof(fen_info->rt->fib6_dst.addr));
17968914add2SVlad Buslov key.ip_version = 6;
17978914add2SVlad Buslov
17988914add2SVlad Buslov /* Can't fail after this point because releasing reference to r
17998914add2SVlad Buslov * requires obtaining sleeping mutex which we can't do in atomic
18008914add2SVlad Buslov * context.
18018914add2SVlad Buslov */
18028914add2SVlad Buslov r = mlx5e_route_lookup_for_update(encap, &key);
18038914add2SVlad Buslov if (!r)
18048914add2SVlad Buslov goto out;
18058914add2SVlad Buslov fib_work->r = r;
18068914add2SVlad Buslov dev_hold(ul_dev);
18078914add2SVlad Buslov
18088914add2SVlad Buslov return fib_work;
18098914add2SVlad Buslov
18108914add2SVlad Buslov out:
18118914add2SVlad Buslov kfree(fib_work);
18128914add2SVlad Buslov return NULL;
18138914add2SVlad Buslov }
18148914add2SVlad Buslov
mlx5e_tc_tun_fib_event(struct notifier_block * nb,unsigned long event,void * ptr)18158914add2SVlad Buslov static int mlx5e_tc_tun_fib_event(struct notifier_block *nb, unsigned long event, void *ptr)
18168914add2SVlad Buslov {
18178914add2SVlad Buslov struct mlx5e_tc_fib_event_data *fib_work;
18188914add2SVlad Buslov struct fib_notifier_info *info = ptr;
18198914add2SVlad Buslov struct mlx5e_tc_tun_encap *encap;
18208914add2SVlad Buslov struct net_device *ul_dev;
18218914add2SVlad Buslov struct mlx5e_priv *priv;
18228914add2SVlad Buslov
18238914add2SVlad Buslov encap = container_of(nb, struct mlx5e_tc_tun_encap, fib_nb);
18248914add2SVlad Buslov priv = encap->priv;
18258914add2SVlad Buslov ul_dev = priv->netdev;
18268914add2SVlad Buslov priv = netdev_priv(ul_dev);
18278914add2SVlad Buslov
18288914add2SVlad Buslov switch (event) {
18298914add2SVlad Buslov case FIB_EVENT_ENTRY_REPLACE:
18308914add2SVlad Buslov case FIB_EVENT_ENTRY_DEL:
18318914add2SVlad Buslov if (info->family == AF_INET)
18328914add2SVlad Buslov fib_work = mlx5e_init_fib_work_ipv4(priv, ul_dev, encap, event, info);
18338914add2SVlad Buslov else if (info->family == AF_INET6)
18348914add2SVlad Buslov fib_work = mlx5e_init_fib_work_ipv6(priv, ul_dev, encap, event, info);
18358914add2SVlad Buslov else
18368914add2SVlad Buslov return NOTIFY_DONE;
18378914add2SVlad Buslov
18388914add2SVlad Buslov if (!IS_ERR_OR_NULL(fib_work)) {
18398914add2SVlad Buslov queue_work(priv->wq, &fib_work->work);
18408914add2SVlad Buslov } else if (IS_ERR(fib_work)) {
18418914add2SVlad Buslov NL_SET_ERR_MSG_MOD(info->extack, "Failed to init fib work");
18428914add2SVlad Buslov mlx5_core_warn(priv->mdev, "Failed to init fib work, %ld\n",
18438914add2SVlad Buslov PTR_ERR(fib_work));
18448914add2SVlad Buslov }
18458914add2SVlad Buslov
18468914add2SVlad Buslov break;
18478914add2SVlad Buslov default:
18488914add2SVlad Buslov return NOTIFY_DONE;
18498914add2SVlad Buslov }
18508914add2SVlad Buslov
18518914add2SVlad Buslov return NOTIFY_DONE;
18528914add2SVlad Buslov }
18538914add2SVlad Buslov
mlx5e_tc_tun_init(struct mlx5e_priv * priv)18548914add2SVlad Buslov struct mlx5e_tc_tun_encap *mlx5e_tc_tun_init(struct mlx5e_priv *priv)
18558914add2SVlad Buslov {
18568914add2SVlad Buslov struct mlx5e_tc_tun_encap *encap;
18578914add2SVlad Buslov int err;
18588914add2SVlad Buslov
18598914add2SVlad Buslov encap = kvzalloc(sizeof(*encap), GFP_KERNEL);
18608914add2SVlad Buslov if (!encap)
18618914add2SVlad Buslov return ERR_PTR(-ENOMEM);
18628914add2SVlad Buslov
18638914add2SVlad Buslov encap->priv = priv;
18648914add2SVlad Buslov encap->fib_nb.notifier_call = mlx5e_tc_tun_fib_event;
18658914add2SVlad Buslov spin_lock_init(&encap->route_lock);
18668914add2SVlad Buslov hash_init(encap->route_tbl);
18678914add2SVlad Buslov err = register_fib_notifier(dev_net(priv->netdev), &encap->fib_nb,
18688914add2SVlad Buslov NULL, NULL);
18698914add2SVlad Buslov if (err) {
18708914add2SVlad Buslov kvfree(encap);
18718914add2SVlad Buslov return ERR_PTR(err);
18728914add2SVlad Buslov }
18738914add2SVlad Buslov
18748914add2SVlad Buslov return encap;
18758914add2SVlad Buslov }
18768914add2SVlad Buslov
mlx5e_tc_tun_cleanup(struct mlx5e_tc_tun_encap * encap)18778914add2SVlad Buslov void mlx5e_tc_tun_cleanup(struct mlx5e_tc_tun_encap *encap)
18788914add2SVlad Buslov {
18798914add2SVlad Buslov if (!encap)
18808914add2SVlad Buslov return;
18818914add2SVlad Buslov
18828914add2SVlad Buslov unregister_fib_notifier(dev_net(encap->priv->netdev), &encap->fib_nb);
18838914add2SVlad Buslov flush_workqueue(encap->priv->wq); /* flush fib event works */
18848914add2SVlad Buslov kvfree(encap);
18858914add2SVlad Buslov }
1886