1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2019 Mellanox Technologies.
3 
4 #include <linux/mlx5/fs.h>
5 #include "eswitch.h"
6 #include "en_tc.h"
7 #include "fs_core.h"
8 
9 struct mlx5_termtbl_handle {
10 	struct hlist_node termtbl_hlist;
11 
12 	struct mlx5_flow_table *termtbl;
13 	struct mlx5_flow_act flow_act;
14 	struct mlx5_flow_destination dest;
15 
16 	struct mlx5_flow_handle *rule;
17 	int ref_count;
18 };
19 
20 static u32
mlx5_eswitch_termtbl_hash(struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest)21 mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act,
22 			  struct mlx5_flow_destination *dest)
23 {
24 	u32 hash;
25 
26 	hash = jhash_1word(flow_act->action, 0);
27 	hash = jhash((const void *)&flow_act->vlan,
28 		     sizeof(flow_act->vlan), hash);
29 	hash = jhash((const void *)&dest->vport.num,
30 		     sizeof(dest->vport.num), hash);
31 	hash = jhash((const void *)&dest->vport.vhca_id,
32 		     sizeof(dest->vport.num), hash);
33 	if (flow_act->pkt_reformat)
34 		hash = jhash(flow_act->pkt_reformat,
35 			     sizeof(*flow_act->pkt_reformat),
36 			     hash);
37 	return hash;
38 }
39 
40 static int
mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act * flow_act1,struct mlx5_flow_destination * dest1,struct mlx5_flow_act * flow_act2,struct mlx5_flow_destination * dest2)41 mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1,
42 			 struct mlx5_flow_destination *dest1,
43 			 struct mlx5_flow_act *flow_act2,
44 			 struct mlx5_flow_destination *dest2)
45 {
46 	int ret;
47 
48 	ret = flow_act1->action != flow_act2->action ||
49 	      dest1->vport.num != dest2->vport.num ||
50 	      dest1->vport.vhca_id != dest2->vport.vhca_id ||
51 	      memcmp(&flow_act1->vlan, &flow_act2->vlan,
52 		     sizeof(flow_act1->vlan));
53 	if (ret)
54 		return ret;
55 
56 	if (flow_act1->pkt_reformat && flow_act2->pkt_reformat)
57 		return memcmp(flow_act1->pkt_reformat, flow_act2->pkt_reformat,
58 			      sizeof(*flow_act1->pkt_reformat));
59 
60 	return !(flow_act1->pkt_reformat == flow_act2->pkt_reformat);
61 }
62 
63 static int
mlx5_eswitch_termtbl_create(struct mlx5_core_dev * dev,struct mlx5_termtbl_handle * tt,struct mlx5_flow_act * flow_act)64 mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev,
65 			    struct mlx5_termtbl_handle *tt,
66 			    struct mlx5_flow_act *flow_act)
67 {
68 	struct mlx5_flow_table_attr ft_attr = {};
69 	struct mlx5_flow_namespace *root_ns;
70 	int err, err2;
71 
72 	root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
73 	if (!root_ns) {
74 		esw_warn(dev, "Failed to get FDB flow namespace\n");
75 		return -EOPNOTSUPP;
76 	}
77 
78 	/* As this is the terminating action then the termination table is the
79 	 * same prio as the slow path
80 	 */
81 	ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED |
82 			MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
83 	ft_attr.prio = FDB_TC_OFFLOAD;
84 	ft_attr.max_fte = 1;
85 	ft_attr.level = 1;
86 	ft_attr.autogroup.max_num_groups = 1;
87 	tt->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr);
88 	if (IS_ERR(tt->termtbl)) {
89 		err = PTR_ERR(tt->termtbl);
90 		esw_warn(dev, "Failed to create termination table, err %pe\n", tt->termtbl);
91 		return err;
92 	}
93 
94 	tt->rule = mlx5_add_flow_rules(tt->termtbl, NULL, flow_act,
95 				       &tt->dest, 1);
96 	if (IS_ERR(tt->rule)) {
97 		err = PTR_ERR(tt->rule);
98 		esw_warn(dev, "Failed to create termination table rule, err %pe\n", tt->rule);
99 		goto add_flow_err;
100 	}
101 	return 0;
102 
103 add_flow_err:
104 	err2 = mlx5_destroy_flow_table(tt->termtbl);
105 	if (err2)
106 		esw_warn(dev, "Failed to destroy termination table, err %d\n", err2);
107 
108 	return err;
109 }
110 
111 static struct mlx5_termtbl_handle *
mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch * esw,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest,struct mlx5_esw_flow_attr * attr)112 mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch *esw,
113 				struct mlx5_flow_act *flow_act,
114 				struct mlx5_flow_destination *dest,
115 				struct mlx5_esw_flow_attr *attr)
116 {
117 	struct mlx5_termtbl_handle *tt;
118 	bool found = false;
119 	u32 hash_key;
120 	int err;
121 
122 	mutex_lock(&esw->offloads.termtbl_mutex);
123 	hash_key = mlx5_eswitch_termtbl_hash(flow_act, dest);
124 	hash_for_each_possible(esw->offloads.termtbl_tbl, tt,
125 			       termtbl_hlist, hash_key) {
126 		if (!mlx5_eswitch_termtbl_cmp(&tt->flow_act, &tt->dest,
127 					      flow_act, dest)) {
128 			found = true;
129 			break;
130 		}
131 	}
132 	if (found)
133 		goto tt_add_ref;
134 
135 	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
136 	if (!tt) {
137 		err = -ENOMEM;
138 		goto tt_create_err;
139 	}
140 
141 	tt->dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
142 	tt->dest.vport.num = dest->vport.num;
143 	tt->dest.vport.vhca_id = dest->vport.vhca_id;
144 	tt->dest.vport.flags = dest->vport.flags;
145 	memcpy(&tt->flow_act, flow_act, sizeof(*flow_act));
146 
147 	err = mlx5_eswitch_termtbl_create(esw->dev, tt, flow_act);
148 	if (err)
149 		goto tt_create_err;
150 
151 	hash_add(esw->offloads.termtbl_tbl, &tt->termtbl_hlist, hash_key);
152 tt_add_ref:
153 	tt->ref_count++;
154 	mutex_unlock(&esw->offloads.termtbl_mutex);
155 	return tt;
156 tt_create_err:
157 	kfree(tt);
158 	mutex_unlock(&esw->offloads.termtbl_mutex);
159 	return ERR_PTR(err);
160 }
161 
162 void
mlx5_eswitch_termtbl_put(struct mlx5_eswitch * esw,struct mlx5_termtbl_handle * tt)163 mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw,
164 			 struct mlx5_termtbl_handle *tt)
165 {
166 	mutex_lock(&esw->offloads.termtbl_mutex);
167 	if (--tt->ref_count == 0)
168 		hash_del(&tt->termtbl_hlist);
169 	mutex_unlock(&esw->offloads.termtbl_mutex);
170 
171 	if (!tt->ref_count) {
172 		mlx5_del_flow_rules(tt->rule);
173 		mlx5_destroy_flow_table(tt->termtbl);
174 		kfree(tt);
175 	}
176 }
177 
178 static void
mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act * src,struct mlx5_flow_act * dst)179 mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src,
180 				  struct mlx5_flow_act *dst)
181 {
182 	if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
183 		src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
184 		dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
185 		memcpy(&dst->vlan[0], &src->vlan[0], sizeof(src->vlan[0]));
186 		memset(&src->vlan[0], 0, sizeof(src->vlan[0]));
187 
188 		if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
189 			src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
190 			dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
191 			memcpy(&dst->vlan[1], &src->vlan[1], sizeof(src->vlan[1]));
192 			memset(&src->vlan[1], 0, sizeof(src->vlan[1]));
193 		}
194 	}
195 }
196 
mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch * esw,const struct mlx5_flow_spec * spec)197 static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw,
198 						const struct mlx5_flow_spec *spec)
199 {
200 	u16 port_mask, port_value;
201 
202 	if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
203 		return spec->flow_context.flow_source ==
204 					MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
205 
206 	port_mask = MLX5_GET(fte_match_param, spec->match_criteria,
207 			     misc_parameters.source_port);
208 	port_value = MLX5_GET(fte_match_param, spec->match_value,
209 			      misc_parameters.source_port);
210 	return (port_mask & port_value) == MLX5_VPORT_UPLINK;
211 }
212 
213 bool
mlx5_eswitch_termtbl_required(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act,struct mlx5_flow_spec * spec)214 mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw,
215 			      struct mlx5_flow_attr *attr,
216 			      struct mlx5_flow_act *flow_act,
217 			      struct mlx5_flow_spec *spec)
218 {
219 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
220 	int i;
221 
222 	if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) ||
223 	    !MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level) ||
224 	    mlx5e_tc_attr_flags_skip(attr->flags) ||
225 	    (!mlx5_eswitch_offload_is_uplink_port(esw, spec) && !esw_attr->int_port))
226 		return false;
227 
228 	/* push vlan on RX */
229 	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
230 	    !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) &
231 	      MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX))
232 		return true;
233 
234 	/* hairpin */
235 	for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
236 		if (!esw_attr->dest_int_port && esw_attr->dests[i].vport_valid &&
237 		    esw_attr->dests[i].vport == MLX5_VPORT_UPLINK)
238 			return true;
239 
240 	return false;
241 }
242 
243 struct mlx5_flow_handle *
mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest,int num_dest)244 mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw,
245 			      struct mlx5_flow_table *fdb,
246 			      struct mlx5_flow_spec *spec,
247 			      struct mlx5_esw_flow_attr *attr,
248 			      struct mlx5_flow_act *flow_act,
249 			      struct mlx5_flow_destination *dest,
250 			      int num_dest)
251 {
252 	struct mlx5_flow_act term_tbl_act = {};
253 	struct mlx5_flow_handle *rule = NULL;
254 	bool term_table_created = false;
255 	int num_vport_dests = 0;
256 	int i, curr_dest;
257 
258 	mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act);
259 	term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
260 
261 	for (i = 0; i < num_dest; i++) {
262 		struct mlx5_termtbl_handle *tt;
263 
264 		/* only vport destinations can be terminated */
265 		if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT)
266 			continue;
267 
268 		if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) {
269 			term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
270 			term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat;
271 		} else {
272 			term_tbl_act.action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
273 			term_tbl_act.pkt_reformat = NULL;
274 		}
275 
276 		/* get the terminating table for the action list */
277 		tt = mlx5_eswitch_termtbl_get_create(esw, &term_tbl_act,
278 						     &dest[i], attr);
279 		if (IS_ERR(tt)) {
280 			esw_warn(esw->dev, "Failed to get termination table, err %pe\n", tt);
281 			goto revert_changes;
282 		}
283 		attr->dests[num_vport_dests].termtbl = tt;
284 		num_vport_dests++;
285 
286 		/* link the destination with the termination table */
287 		dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
288 		dest[i].ft = tt->termtbl;
289 		term_table_created = true;
290 	}
291 
292 	/* at least one destination should reference a termination table */
293 	if (!term_table_created)
294 		goto revert_changes;
295 
296 	/* create the FTE */
297 	flow_act->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
298 	flow_act->pkt_reformat = NULL;
299 	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
300 	rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
301 	if (IS_ERR(rule))
302 		goto revert_changes;
303 
304 	goto out;
305 
306 revert_changes:
307 	/* revert the changes that were made to the original flow_act
308 	 * and fall-back to the original rule actions
309 	 */
310 	mlx5_eswitch_termtbl_actions_move(&term_tbl_act, flow_act);
311 
312 	for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) {
313 		struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl;
314 
315 		attr->dests[curr_dest].termtbl = NULL;
316 
317 		/* search for the destination associated with the
318 		 * current term table
319 		 */
320 		for (i = 0; i < num_dest; i++) {
321 			if (dest[i].ft != tt->termtbl)
322 				continue;
323 
324 			memset(&dest[i], 0, sizeof(dest[i]));
325 			dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
326 			dest[i].vport.num = tt->dest.vport.num;
327 			dest[i].vport.vhca_id = tt->dest.vport.vhca_id;
328 			mlx5_eswitch_termtbl_put(esw, tt);
329 			break;
330 		}
331 	}
332 	rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
333 out:
334 	return rule;
335 }
336