1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
43 #include "rdma.h"
44 #include "en.h"
45 #include "fs_core.h"
46 #include "lib/devcom.h"
47 #include "lib/eq.h"
48 #include "lib/fs_chains.h"
49 #include "en_tc.h"
50 #include "en/mapping.h"
51 #include "devlink.h"
52 #include "lag/lag.h"
53 #include "en/tc/post_meter.h"
54
55 /* There are two match-all miss flows, one for unicast dst mac and
56 * one for multicast.
57 */
58 #define MLX5_ESW_MISS_FLOWS (2)
59 #define UPLINK_REP_INDEX 0
60
61 #define MLX5_ESW_VPORT_TBL_SIZE 128
62 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
63
64 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
65
66 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
67 .max_fte = MLX5_ESW_VPORT_TBL_SIZE,
68 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
69 .flags = 0,
70 };
71
mlx5_eswitch_get_rep(struct mlx5_eswitch * esw,u16 vport_num)72 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
73 u16 vport_num)
74 {
75 return xa_load(&esw->offloads.vport_reps, vport_num);
76 }
77
78 static void
mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr)79 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
80 struct mlx5_flow_spec *spec,
81 struct mlx5_esw_flow_attr *attr)
82 {
83 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
84 return;
85
86 if (attr->int_port) {
87 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port);
88
89 return;
90 }
91
92 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
93 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
94 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
95 }
96
97 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
98 * are not needed as well in the following process. So clear them all for simplicity.
99 */
100 void
mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec)101 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
102 {
103 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
104 void *misc2;
105
106 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
107 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
108
109 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
110 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
111
112 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
113 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
114 }
115 }
116
117 static void
mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,struct mlx5_eswitch * src_esw,u16 vport)118 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
119 struct mlx5_flow_spec *spec,
120 struct mlx5_flow_attr *attr,
121 struct mlx5_eswitch *src_esw,
122 u16 vport)
123 {
124 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
125 u32 metadata;
126 void *misc2;
127 void *misc;
128
129 /* Use metadata matching because vport is not represented by single
130 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
131 */
132 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
133 if (mlx5_esw_indir_table_decap_vport(attr))
134 vport = mlx5_esw_indir_table_decap_vport(attr);
135
136 if (!attr->chain && esw_attr && esw_attr->int_port)
137 metadata =
138 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port);
139 else
140 metadata =
141 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport);
142
143 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
144 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
145
146 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
147 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
148 mlx5_eswitch_get_vport_metadata_mask());
149
150 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
151 } else {
152 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
153 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
154
155 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
156 MLX5_SET(fte_match_set_misc, misc,
157 source_eswitch_owner_vhca_id,
158 MLX5_CAP_GEN(src_esw->dev, vhca_id));
159
160 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
161 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
162 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
163 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
164 source_eswitch_owner_vhca_id);
165
166 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
167 }
168 }
169
170 static int
esw_setup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)171 esw_setup_decap_indir(struct mlx5_eswitch *esw,
172 struct mlx5_flow_attr *attr)
173 {
174 struct mlx5_flow_table *ft;
175
176 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
177 return -EOPNOTSUPP;
178
179 ft = mlx5_esw_indir_table_get(esw, attr,
180 mlx5_esw_indir_table_decap_vport(attr), true);
181 return PTR_ERR_OR_ZERO(ft);
182 }
183
184 static void
esw_cleanup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)185 esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
186 struct mlx5_flow_attr *attr)
187 {
188 if (mlx5_esw_indir_table_decap_vport(attr))
189 mlx5_esw_indir_table_put(esw,
190 mlx5_esw_indir_table_decap_vport(attr),
191 true);
192 }
193
194 static int
esw_setup_mtu_dest(struct mlx5_flow_destination * dest,struct mlx5e_meter_attr * meter,int i)195 esw_setup_mtu_dest(struct mlx5_flow_destination *dest,
196 struct mlx5e_meter_attr *meter,
197 int i)
198 {
199 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE;
200 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN;
201 dest[i].range.min = 0;
202 dest[i].range.max = meter->params.mtu;
203 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter);
204 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter);
205
206 return 0;
207 }
208
209 static int
esw_setup_sampler_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,u32 sampler_id,int i)210 esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
211 struct mlx5_flow_act *flow_act,
212 u32 sampler_id,
213 int i)
214 {
215 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
216 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
217 dest[i].sampler_id = sampler_id;
218
219 return 0;
220 }
221
222 static int
esw_setup_ft_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int i)223 esw_setup_ft_dest(struct mlx5_flow_destination *dest,
224 struct mlx5_flow_act *flow_act,
225 struct mlx5_eswitch *esw,
226 struct mlx5_flow_attr *attr,
227 int i)
228 {
229 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
230 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
231 dest[i].ft = attr->dest_ft;
232
233 if (mlx5_esw_indir_table_decap_vport(attr))
234 return esw_setup_decap_indir(esw, attr);
235 return 0;
236 }
237
238 static void
esw_setup_accept_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,int i)239 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
240 struct mlx5_fs_chains *chains, int i)
241 {
242 if (mlx5_chains_ignore_flow_level_supported(chains))
243 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
244 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
245 dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
246 }
247
248 static void
esw_setup_slow_path_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,int i)249 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
250 struct mlx5_eswitch *esw, int i)
251 {
252 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
253 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
254 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
255 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw);
256 }
257
258 static int
esw_setup_chain_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level,int i)259 esw_setup_chain_dest(struct mlx5_flow_destination *dest,
260 struct mlx5_flow_act *flow_act,
261 struct mlx5_fs_chains *chains,
262 u32 chain, u32 prio, u32 level,
263 int i)
264 {
265 struct mlx5_flow_table *ft;
266
267 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
268 ft = mlx5_chains_get_table(chains, chain, prio, level);
269 if (IS_ERR(ft))
270 return PTR_ERR(ft);
271
272 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
273 dest[i].ft = ft;
274 return 0;
275 }
276
esw_put_dest_tables_loop(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int from,int to)277 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
278 int from, int to)
279 {
280 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
281 struct mlx5_fs_chains *chains = esw_chains(esw);
282 int i;
283
284 for (i = from; i < to; i++)
285 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
286 mlx5_chains_put_table(chains, 0, 1, 0);
287 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport,
288 esw_attr->dests[i].mdev))
289 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false);
290 }
291
292 static bool
esw_is_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)293 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
294 {
295 int i;
296
297 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
298 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
299 return true;
300 return false;
301 }
302
303 static int
esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains,struct mlx5_flow_attr * attr,int * i)304 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
305 struct mlx5_flow_act *flow_act,
306 struct mlx5_eswitch *esw,
307 struct mlx5_fs_chains *chains,
308 struct mlx5_flow_attr *attr,
309 int *i)
310 {
311 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
312 int err;
313
314 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
315 return -EOPNOTSUPP;
316
317 /* flow steering cannot handle more than one dest with the same ft
318 * in a single flow
319 */
320 if (esw_attr->out_count - esw_attr->split_count > 1)
321 return -EOPNOTSUPP;
322
323 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
324 if (err)
325 return err;
326
327 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
328 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
329 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
330 }
331 (*i)++;
332
333 return 0;
334 }
335
esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)336 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
337 struct mlx5_flow_attr *attr)
338 {
339 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
340
341 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
342 }
343
344 static bool
esw_is_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)345 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
346 {
347 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
348 bool result = false;
349 int i;
350
351 /* Indirect table is supported only for flows with in_port uplink
352 * and the destination is vport on the same eswitch as the uplink,
353 * return false in case at least one of destinations doesn't meet
354 * this criteria.
355 */
356 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
357 if (esw_attr->dests[i].vport_valid &&
358 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport,
359 esw_attr->dests[i].mdev)) {
360 result = true;
361 } else {
362 result = false;
363 break;
364 }
365 }
366 return result;
367 }
368
369 static int
esw_setup_indir_table(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int * i)370 esw_setup_indir_table(struct mlx5_flow_destination *dest,
371 struct mlx5_flow_act *flow_act,
372 struct mlx5_eswitch *esw,
373 struct mlx5_flow_attr *attr,
374 int *i)
375 {
376 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
377 int j, err;
378
379 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
380 return -EOPNOTSUPP;
381
382 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
383 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
384 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
385
386 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr,
387 esw_attr->dests[j].vport, false);
388 if (IS_ERR(dest[*i].ft)) {
389 err = PTR_ERR(dest[*i].ft);
390 goto err_indir_tbl_get;
391 }
392 }
393
394 if (mlx5_esw_indir_table_decap_vport(attr)) {
395 err = esw_setup_decap_indir(esw, attr);
396 if (err)
397 goto err_indir_tbl_get;
398 }
399
400 return 0;
401
402 err_indir_tbl_get:
403 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
404 return err;
405 }
406
esw_cleanup_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)407 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
408 {
409 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
410
411 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
412 esw_cleanup_decap_indir(esw, attr);
413 }
414
415 static void
esw_cleanup_chain_dest(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)416 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
417 {
418 mlx5_chains_put_table(chains, chain, prio, level);
419 }
420
esw_same_vhca_id(struct mlx5_core_dev * mdev1,struct mlx5_core_dev * mdev2)421 static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2)
422 {
423 return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id);
424 }
425
esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx)426 static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw,
427 struct mlx5_esw_flow_attr *esw_attr,
428 int attr_idx)
429 {
430 if (esw->offloads.ft_ipsec_tx_pol &&
431 esw_attr->dests[attr_idx].vport_valid &&
432 esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK &&
433 /* To be aligned with software, encryption is needed only for tunnel device */
434 (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) &&
435 esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport &&
436 esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev))
437 return true;
438
439 return false;
440 }
441
esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)442 static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw,
443 struct mlx5_esw_flow_attr *esw_attr)
444 {
445 int i;
446
447 if (!esw->offloads.ft_ipsec_tx_pol)
448 return true;
449
450 for (i = 0; i < esw_attr->split_count; i++)
451 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i))
452 return false;
453
454 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
455 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) &&
456 (esw_attr->out_count - esw_attr->split_count > 1))
457 return false;
458
459 return true;
460 }
461
462 static void
esw_setup_dest_fwd_vport(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)463 esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
464 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
465 int attr_idx, int dest_idx, bool pkt_reformat)
466 {
467 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
468 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport;
469 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
470 dest[dest_idx].vport.vhca_id =
471 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
472 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
473 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
474 mlx5_lag_is_mpesw(esw->dev))
475 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
476 }
477 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
478 if (pkt_reformat) {
479 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
480 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
481 }
482 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
483 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
484 }
485 }
486
487 static void
esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)488 esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
489 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
490 int attr_idx, int dest_idx, bool pkt_reformat)
491 {
492 dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol;
493 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
494 if (pkt_reformat &&
495 esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
496 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
497 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
498 }
499 }
500
501 static void
esw_setup_vport_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)502 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
503 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
504 int attr_idx, int dest_idx, bool pkt_reformat)
505 {
506 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
507 esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr,
508 attr_idx, dest_idx, pkt_reformat);
509 else
510 esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr,
511 attr_idx, dest_idx, pkt_reformat);
512 }
513
514 static int
esw_setup_vport_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int i)515 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
516 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
517 int i)
518 {
519 int j;
520
521 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
522 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
523 return i;
524 }
525
526 static bool
esw_src_port_rewrite_supported(struct mlx5_eswitch * esw)527 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
528 {
529 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
530 mlx5_eswitch_vport_match_metadata_enabled(esw) &&
531 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
532 }
533
534 static bool
esw_dests_to_int_external(struct mlx5_flow_destination * dests,int max_dest)535 esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest)
536 {
537 bool internal_dest = false, external_dest = false;
538 int i;
539
540 for (i = 0; i < max_dest; i++) {
541 if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT &&
542 dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK)
543 continue;
544
545 /* Uplink dest is external, but considered as internal
546 * if there is reformat because firmware uses LB+hairpin to support it.
547 */
548 if (dests[i].vport.num == MLX5_VPORT_UPLINK &&
549 !(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID))
550 external_dest = true;
551 else
552 internal_dest = true;
553
554 if (internal_dest && external_dest)
555 return true;
556 }
557
558 return false;
559 }
560
561 static int
esw_setup_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int * i)562 esw_setup_dests(struct mlx5_flow_destination *dest,
563 struct mlx5_flow_act *flow_act,
564 struct mlx5_eswitch *esw,
565 struct mlx5_flow_attr *attr,
566 struct mlx5_flow_spec *spec,
567 int *i)
568 {
569 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
570 struct mlx5_fs_chains *chains = esw_chains(esw);
571 int err = 0;
572
573 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
574 esw_src_port_rewrite_supported(esw))
575 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
576
577 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
578 esw_setup_slow_path_dest(dest, flow_act, esw, *i);
579 (*i)++;
580 goto out;
581 }
582
583 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) {
584 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i);
585 (*i)++;
586 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
587 esw_setup_accept_dest(dest, flow_act, chains, *i);
588 (*i)++;
589 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) {
590 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i);
591 (*i)++;
592 } else if (esw_is_indir_table(esw, attr)) {
593 err = esw_setup_indir_table(dest, flow_act, esw, attr, i);
594 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
595 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
596 } else {
597 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
598
599 if (attr->dest_ft) {
600 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i);
601 (*i)++;
602 } else if (attr->dest_chain) {
603 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
604 1, 0, *i);
605 (*i)++;
606 }
607 }
608
609 out:
610 return err;
611 }
612
613 static void
esw_cleanup_dests(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)614 esw_cleanup_dests(struct mlx5_eswitch *esw,
615 struct mlx5_flow_attr *attr)
616 {
617 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
618 struct mlx5_fs_chains *chains = esw_chains(esw);
619
620 if (attr->dest_ft) {
621 esw_cleanup_decap_indir(esw, attr);
622 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
623 if (attr->dest_chain)
624 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
625 else if (esw_is_indir_table(esw, attr))
626 esw_cleanup_indir_table(esw, attr);
627 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
628 esw_cleanup_chain_src_port_rewrite(esw, attr);
629 }
630 }
631
632 static void
esw_setup_meter(struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act)633 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act)
634 {
635 struct mlx5e_flow_meter_handle *meter;
636
637 meter = attr->meter_attr.meter;
638 flow_act->exe_aso.type = attr->exe_aso_type;
639 flow_act->exe_aso.object_id = meter->obj_id;
640 flow_act->exe_aso.flow_meter.meter_idx = meter->idx;
641 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN;
642 /* use metadata reg 5 for packet color */
643 flow_act->exe_aso.return_reg_id = 5;
644 }
645
646 struct mlx5_flow_handle *
mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)647 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
648 struct mlx5_flow_spec *spec,
649 struct mlx5_flow_attr *attr)
650 {
651 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
652 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
653 struct mlx5_fs_chains *chains = esw_chains(esw);
654 bool split = !!(esw_attr->split_count);
655 struct mlx5_vport_tbl_attr fwd_attr;
656 struct mlx5_flow_destination *dest;
657 struct mlx5_flow_handle *rule;
658 struct mlx5_flow_table *fdb;
659 int i = 0;
660
661 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
662 return ERR_PTR(-EOPNOTSUPP);
663
664 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
665 return ERR_PTR(-EOPNOTSUPP);
666
667 if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr))
668 return ERR_PTR(-EOPNOTSUPP);
669
670 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
671 if (!dest)
672 return ERR_PTR(-ENOMEM);
673
674 flow_act.action = attr->action;
675
676 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
677 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
678 flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
679 flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
680 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
681 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
682 flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
683 flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
684 }
685 }
686
687 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
688
689 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
690 int err;
691
692 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
693 if (err) {
694 rule = ERR_PTR(err);
695 goto err_create_goto_table;
696 }
697
698 /* Header rewrite with combined wire+loopback in FDB is not allowed */
699 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) &&
700 esw_dests_to_int_external(dest, i)) {
701 esw_warn(esw->dev,
702 "FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n");
703 rule = ERR_PTR(-EINVAL);
704 goto err_esw_get;
705 }
706 }
707
708 if (esw_attr->decap_pkt_reformat)
709 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
710
711 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
712 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
713 dest[i].counter_id = mlx5_fc_id(attr->counter);
714 i++;
715 }
716
717 if (attr->outer_match_level != MLX5_MATCH_NONE)
718 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
719 if (attr->inner_match_level != MLX5_MATCH_NONE)
720 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
721
722 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
723 flow_act.modify_hdr = attr->modify_hdr;
724
725 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
726 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)
727 esw_setup_meter(attr, &flow_act);
728
729 if (split) {
730 fwd_attr.chain = attr->chain;
731 fwd_attr.prio = attr->prio;
732 fwd_attr.vport = esw_attr->in_rep->vport;
733 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
734
735 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
736 } else {
737 if (attr->chain || attr->prio)
738 fdb = mlx5_chains_get_table(chains, attr->chain,
739 attr->prio, 0);
740 else
741 fdb = attr->ft;
742
743 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
744 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
745 esw_attr->in_mdev->priv.eswitch,
746 esw_attr->in_rep->vport);
747 }
748 if (IS_ERR(fdb)) {
749 rule = ERR_CAST(fdb);
750 goto err_esw_get;
751 }
752
753 if (!i) {
754 kfree(dest);
755 dest = NULL;
756 }
757
758 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
759 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
760 &flow_act, dest, i);
761 else
762 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
763 if (IS_ERR(rule))
764 goto err_add_rule;
765 else
766 atomic64_inc(&esw->offloads.num_flows);
767
768 kfree(dest);
769 return rule;
770
771 err_add_rule:
772 if (split)
773 mlx5_esw_vporttbl_put(esw, &fwd_attr);
774 else if (attr->chain || attr->prio)
775 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
776 err_esw_get:
777 esw_cleanup_dests(esw, attr);
778 err_create_goto_table:
779 kfree(dest);
780 return rule;
781 }
782
783 struct mlx5_flow_handle *
mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)784 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
785 struct mlx5_flow_spec *spec,
786 struct mlx5_flow_attr *attr)
787 {
788 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
789 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
790 struct mlx5_fs_chains *chains = esw_chains(esw);
791 struct mlx5_vport_tbl_attr fwd_attr;
792 struct mlx5_flow_destination *dest;
793 struct mlx5_flow_table *fast_fdb;
794 struct mlx5_flow_table *fwd_fdb;
795 struct mlx5_flow_handle *rule;
796 int i, err = 0;
797
798 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
799 if (!dest)
800 return ERR_PTR(-ENOMEM);
801
802 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
803 if (IS_ERR(fast_fdb)) {
804 rule = ERR_CAST(fast_fdb);
805 goto err_get_fast;
806 }
807
808 fwd_attr.chain = attr->chain;
809 fwd_attr.prio = attr->prio;
810 fwd_attr.vport = esw_attr->in_rep->vport;
811 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
812 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
813 if (IS_ERR(fwd_fdb)) {
814 rule = ERR_CAST(fwd_fdb);
815 goto err_get_fwd;
816 }
817
818 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
819 for (i = 0; i < esw_attr->split_count; i++) {
820 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
821 /* Source port rewrite (forward to ovs internal port or statck device) isn't
822 * supported in the rule of split action.
823 */
824 err = -EOPNOTSUPP;
825 else
826 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
827
828 if (err) {
829 rule = ERR_PTR(err);
830 goto err_chain_src_rewrite;
831 }
832 }
833 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
834 dest[i].ft = fwd_fdb;
835 i++;
836
837 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
838 esw_attr->in_mdev->priv.eswitch,
839 esw_attr->in_rep->vport);
840
841 if (attr->outer_match_level != MLX5_MATCH_NONE)
842 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
843
844 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
845 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
846
847 if (IS_ERR(rule)) {
848 i = esw_attr->split_count;
849 goto err_chain_src_rewrite;
850 }
851
852 atomic64_inc(&esw->offloads.num_flows);
853
854 kfree(dest);
855 return rule;
856 err_chain_src_rewrite:
857 mlx5_esw_vporttbl_put(esw, &fwd_attr);
858 err_get_fwd:
859 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
860 err_get_fast:
861 kfree(dest);
862 return rule;
863 }
864
865 static void
__mlx5_eswitch_del_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr,bool fwd_rule)866 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
867 struct mlx5_flow_handle *rule,
868 struct mlx5_flow_attr *attr,
869 bool fwd_rule)
870 {
871 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
872 struct mlx5_fs_chains *chains = esw_chains(esw);
873 bool split = (esw_attr->split_count > 0);
874 struct mlx5_vport_tbl_attr fwd_attr;
875 int i;
876
877 mlx5_del_flow_rules(rule);
878
879 if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
880 /* unref the term table */
881 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
882 if (esw_attr->dests[i].termtbl)
883 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
884 }
885 }
886
887 atomic64_dec(&esw->offloads.num_flows);
888
889 if (fwd_rule || split) {
890 fwd_attr.chain = attr->chain;
891 fwd_attr.prio = attr->prio;
892 fwd_attr.vport = esw_attr->in_rep->vport;
893 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
894 }
895
896 if (fwd_rule) {
897 mlx5_esw_vporttbl_put(esw, &fwd_attr);
898 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
899 } else {
900 if (split)
901 mlx5_esw_vporttbl_put(esw, &fwd_attr);
902 else if (attr->chain || attr->prio)
903 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
904 esw_cleanup_dests(esw, attr);
905 }
906 }
907
908 void
mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)909 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
910 struct mlx5_flow_handle *rule,
911 struct mlx5_flow_attr *attr)
912 {
913 __mlx5_eswitch_del_rule(esw, rule, attr, false);
914 }
915
916 void
mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)917 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
918 struct mlx5_flow_handle *rule,
919 struct mlx5_flow_attr *attr)
920 {
921 __mlx5_eswitch_del_rule(esw, rule, attr, true);
922 }
923
924 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch * on_esw,struct mlx5_eswitch * from_esw,struct mlx5_eswitch_rep * rep,u32 sqn)925 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
926 struct mlx5_eswitch *from_esw,
927 struct mlx5_eswitch_rep *rep,
928 u32 sqn)
929 {
930 struct mlx5_flow_act flow_act = {0};
931 struct mlx5_flow_destination dest = {};
932 struct mlx5_flow_handle *flow_rule;
933 struct mlx5_flow_spec *spec;
934 void *misc;
935 u16 vport;
936
937 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
938 if (!spec) {
939 flow_rule = ERR_PTR(-ENOMEM);
940 goto out;
941 }
942
943 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
944 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
945
946 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
947 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
948
949 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
950
951 /* source vport is the esw manager */
952 vport = from_esw->manager_vport;
953
954 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) {
955 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
956 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
957 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport));
958
959 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
960 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
961 mlx5_eswitch_get_vport_metadata_mask());
962
963 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
964 } else {
965 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
966 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
967
968 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
969 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
970 MLX5_CAP_GEN(from_esw->dev, vhca_id));
971
972 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
973 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
974
975 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
976 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
977 source_eswitch_owner_vhca_id);
978
979 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
980 }
981
982 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
983 dest.vport.num = rep->vport;
984 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
985 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
986 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
987
988 if (rep->vport == MLX5_VPORT_UPLINK &&
989 on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) {
990 dest.ft = on_esw->offloads.ft_ipsec_tx_pol;
991 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL;
992 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
993 } else {
994 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
995 dest.vport.num = rep->vport;
996 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
997 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
998 }
999
1000 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) &&
1001 rep->vport == MLX5_VPORT_UPLINK)
1002 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1003
1004 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw),
1005 spec, &flow_act, &dest, 1);
1006 if (IS_ERR(flow_rule))
1007 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n",
1008 PTR_ERR(flow_rule));
1009 out:
1010 kvfree(spec);
1011 return flow_rule;
1012 }
1013 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1014
mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle * rule)1015 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1016 {
1017 mlx5_del_flow_rules(rule);
1018 }
1019
mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle * rule)1020 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
1021 {
1022 if (rule)
1023 mlx5_del_flow_rules(rule);
1024 }
1025
1026 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch * esw,u16 vport_num)1027 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
1028 {
1029 struct mlx5_flow_destination dest = {};
1030 struct mlx5_flow_act flow_act = {0};
1031 struct mlx5_flow_handle *flow_rule;
1032 struct mlx5_flow_spec *spec;
1033
1034 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1035 if (!spec)
1036 return ERR_PTR(-ENOMEM);
1037
1038 MLX5_SET(fte_match_param, spec->match_criteria,
1039 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1040 MLX5_SET(fte_match_param, spec->match_criteria,
1041 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1042 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1043 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1044
1045 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1046 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1047 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1048
1049 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1050 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1051 dest.vport.num = vport_num;
1052
1053 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1054 spec, &flow_act, &dest, 1);
1055 if (IS_ERR(flow_rule))
1056 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n",
1057 vport_num, PTR_ERR(flow_rule));
1058
1059 kvfree(spec);
1060 return flow_rule;
1061 }
1062
mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch * esw)1063 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1064 {
1065 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1066 MLX5_FDB_TO_VPORT_REG_C_1;
1067 }
1068
esw_set_passing_vport_metadata(struct mlx5_eswitch * esw,bool enable)1069 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1070 {
1071 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1072 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1073 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1074 u8 curr, wanted;
1075 int err;
1076
1077 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1078 !mlx5_eswitch_vport_match_metadata_enabled(esw))
1079 return 0;
1080
1081 MLX5_SET(query_esw_vport_context_in, in, opcode,
1082 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1083 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1084 if (err)
1085 return err;
1086
1087 curr = MLX5_GET(query_esw_vport_context_out, out,
1088 esw_vport_context.fdb_to_vport_reg_c_id);
1089 wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1090 if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1091 wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1092
1093 if (enable)
1094 curr |= wanted;
1095 else
1096 curr &= ~wanted;
1097
1098 MLX5_SET(modify_esw_vport_context_in, min,
1099 esw_vport_context.fdb_to_vport_reg_c_id, curr);
1100 MLX5_SET(modify_esw_vport_context_in, min,
1101 field_select.fdb_to_vport_reg_c_id, 1);
1102
1103 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
1104 if (!err) {
1105 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1106 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1107 else
1108 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1109 }
1110
1111 return err;
1112 }
1113
peer_miss_rules_setup(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev,struct mlx5_flow_spec * spec,struct mlx5_flow_destination * dest)1114 static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1115 struct mlx5_core_dev *peer_dev,
1116 struct mlx5_flow_spec *spec,
1117 struct mlx5_flow_destination *dest)
1118 {
1119 void *misc;
1120
1121 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1122 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1123 misc_parameters_2);
1124 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1125 mlx5_eswitch_get_vport_metadata_mask());
1126
1127 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1128 } else {
1129 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1130 misc_parameters);
1131
1132 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1133 MLX5_CAP_GEN(peer_dev, vhca_id));
1134
1135 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1136
1137 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1138 misc_parameters);
1139 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1140 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1141 source_eswitch_owner_vhca_id);
1142 }
1143
1144 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1145 dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1146 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1147 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1148 }
1149
esw_set_peer_miss_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,struct mlx5_flow_spec * spec,u16 vport)1150 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1151 struct mlx5_eswitch *peer_esw,
1152 struct mlx5_flow_spec *spec,
1153 u16 vport)
1154 {
1155 void *misc;
1156
1157 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1158 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1159 misc_parameters_2);
1160 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1161 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1162 vport));
1163 } else {
1164 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1165 misc_parameters);
1166 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1167 }
1168 }
1169
esw_add_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1170 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1171 struct mlx5_core_dev *peer_dev)
1172 {
1173 struct mlx5_flow_destination dest = {};
1174 struct mlx5_flow_act flow_act = {0};
1175 struct mlx5_flow_handle **flows;
1176 /* total vports is the same for both e-switches */
1177 int nvports = esw->total_vports;
1178 struct mlx5_flow_handle *flow;
1179 struct mlx5_flow_spec *spec;
1180 struct mlx5_vport *vport;
1181 int err, pfindex;
1182 unsigned long i;
1183 void *misc;
1184
1185 if (!MLX5_VPORT_MANAGER(esw->dev) && !mlx5_core_is_ecpf_esw_manager(esw->dev))
1186 return 0;
1187
1188 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1189 if (!spec)
1190 return -ENOMEM;
1191
1192 peer_miss_rules_setup(esw, peer_dev, spec, &dest);
1193
1194 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL);
1195 if (!flows) {
1196 err = -ENOMEM;
1197 goto alloc_flows_err;
1198 }
1199
1200 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1201 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1202 misc_parameters);
1203
1204 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1205 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1206 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1207 spec, MLX5_VPORT_PF);
1208
1209 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1210 spec, &flow_act, &dest, 1);
1211 if (IS_ERR(flow)) {
1212 err = PTR_ERR(flow);
1213 goto add_pf_flow_err;
1214 }
1215 flows[vport->index] = flow;
1216 }
1217
1218 if (mlx5_ecpf_vport_exists(esw->dev)) {
1219 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1220 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1221 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1222 spec, &flow_act, &dest, 1);
1223 if (IS_ERR(flow)) {
1224 err = PTR_ERR(flow);
1225 goto add_ecpf_flow_err;
1226 }
1227 flows[vport->index] = flow;
1228 }
1229
1230 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1231 esw_set_peer_miss_rule_source_port(esw,
1232 peer_dev->priv.eswitch,
1233 spec, vport->vport);
1234
1235 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1236 spec, &flow_act, &dest, 1);
1237 if (IS_ERR(flow)) {
1238 err = PTR_ERR(flow);
1239 goto add_vf_flow_err;
1240 }
1241 flows[vport->index] = flow;
1242 }
1243
1244 if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1245 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1246 if (i >= mlx5_core_max_ec_vfs(peer_dev))
1247 break;
1248 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1249 spec, vport->vport);
1250 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1251 spec, &flow_act, &dest, 1);
1252 if (IS_ERR(flow)) {
1253 err = PTR_ERR(flow);
1254 goto add_ec_vf_flow_err;
1255 }
1256 flows[vport->index] = flow;
1257 }
1258 }
1259
1260 pfindex = mlx5_get_dev_index(peer_dev);
1261 if (pfindex >= MLX5_MAX_PORTS) {
1262 esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n",
1263 pfindex, MLX5_MAX_PORTS);
1264 err = -EINVAL;
1265 goto add_ec_vf_flow_err;
1266 }
1267 esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows;
1268
1269 kvfree(spec);
1270 return 0;
1271
1272 add_ec_vf_flow_err:
1273 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1274 if (!flows[vport->index])
1275 continue;
1276 mlx5_del_flow_rules(flows[vport->index]);
1277 }
1278 add_vf_flow_err:
1279 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1280 if (!flows[vport->index])
1281 continue;
1282 mlx5_del_flow_rules(flows[vport->index]);
1283 }
1284 if (mlx5_ecpf_vport_exists(esw->dev)) {
1285 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1286 mlx5_del_flow_rules(flows[vport->index]);
1287 }
1288 add_ecpf_flow_err:
1289 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1290 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1291 mlx5_del_flow_rules(flows[vport->index]);
1292 }
1293 add_pf_flow_err:
1294 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1295 kvfree(flows);
1296 alloc_flows_err:
1297 kvfree(spec);
1298 return err;
1299 }
1300
esw_del_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1301 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1302 struct mlx5_core_dev *peer_dev)
1303 {
1304 u16 peer_index = mlx5_get_dev_index(peer_dev);
1305 struct mlx5_flow_handle **flows;
1306 struct mlx5_vport *vport;
1307 unsigned long i;
1308
1309 flows = esw->fdb_table.offloads.peer_miss_rules[peer_index];
1310 if (!flows)
1311 return;
1312
1313 if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1314 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1315 /* The flow for a particular vport could be NULL if the other ECPF
1316 * has fewer or no VFs enabled
1317 */
1318 if (!flows[vport->index])
1319 continue;
1320 mlx5_del_flow_rules(flows[vport->index]);
1321 }
1322 }
1323
1324 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev))
1325 mlx5_del_flow_rules(flows[vport->index]);
1326
1327 if (mlx5_ecpf_vport_exists(esw->dev)) {
1328 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1329 mlx5_del_flow_rules(flows[vport->index]);
1330 }
1331
1332 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1333 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1334 mlx5_del_flow_rules(flows[vport->index]);
1335 }
1336
1337 kvfree(flows);
1338 esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL;
1339 }
1340
esw_add_fdb_miss_rule(struct mlx5_eswitch * esw)1341 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1342 {
1343 struct mlx5_flow_act flow_act = {0};
1344 struct mlx5_flow_destination dest = {};
1345 struct mlx5_flow_handle *flow_rule = NULL;
1346 struct mlx5_flow_spec *spec;
1347 void *headers_c;
1348 void *headers_v;
1349 int err = 0;
1350 u8 *dmac_c;
1351 u8 *dmac_v;
1352
1353 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1354 if (!spec) {
1355 err = -ENOMEM;
1356 goto out;
1357 }
1358
1359 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1360 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1361 outer_headers);
1362 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1363 outer_headers.dmac_47_16);
1364 dmac_c[0] = 0x01;
1365
1366 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1367 dest.vport.num = esw->manager_vport;
1368 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1369
1370 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1371 spec, &flow_act, &dest, 1);
1372 if (IS_ERR(flow_rule)) {
1373 err = PTR_ERR(flow_rule);
1374 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
1375 goto out;
1376 }
1377
1378 esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1379
1380 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1381 outer_headers);
1382 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1383 outer_headers.dmac_47_16);
1384 dmac_v[0] = 0x01;
1385 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1386 spec, &flow_act, &dest, 1);
1387 if (IS_ERR(flow_rule)) {
1388 err = PTR_ERR(flow_rule);
1389 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1390 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1391 goto out;
1392 }
1393
1394 esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1395
1396 out:
1397 kvfree(spec);
1398 return err;
1399 }
1400
1401 struct mlx5_flow_handle *
esw_add_restore_rule(struct mlx5_eswitch * esw,u32 tag)1402 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1403 {
1404 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1405 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1406 struct mlx5_flow_context *flow_context;
1407 struct mlx5_flow_handle *flow_rule;
1408 struct mlx5_flow_destination dest;
1409 struct mlx5_flow_spec *spec;
1410 void *misc;
1411
1412 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1413 return ERR_PTR(-EOPNOTSUPP);
1414
1415 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1416 if (!spec)
1417 return ERR_PTR(-ENOMEM);
1418
1419 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1420 misc_parameters_2);
1421 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1422 ESW_REG_C0_USER_DATA_METADATA_MASK);
1423 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1424 misc_parameters_2);
1425 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1426 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1427 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1428 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1429 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1430
1431 flow_context = &spec->flow_context;
1432 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1433 flow_context->flow_tag = tag;
1434 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1435 dest.ft = esw->offloads.ft_offloads;
1436
1437 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1438 kvfree(spec);
1439
1440 if (IS_ERR(flow_rule))
1441 esw_warn(esw->dev,
1442 "Failed to create restore rule for tag: %d, err(%d)\n",
1443 tag, (int)PTR_ERR(flow_rule));
1444
1445 return flow_rule;
1446 }
1447
1448 #define MAX_PF_SQ 256
1449 #define MAX_SQ_NVPORTS 32
1450
1451 void
mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch * esw,u32 * flow_group_in,int match_params)1452 mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1453 u32 *flow_group_in,
1454 int match_params)
1455 {
1456 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1457 flow_group_in,
1458 match_criteria);
1459
1460 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1461 MLX5_SET(create_flow_group_in, flow_group_in,
1462 match_criteria_enable,
1463 MLX5_MATCH_MISC_PARAMETERS_2 | match_params);
1464
1465 MLX5_SET(fte_match_param, match_criteria,
1466 misc_parameters_2.metadata_reg_c_0,
1467 mlx5_eswitch_get_vport_metadata_mask());
1468 } else {
1469 MLX5_SET(create_flow_group_in, flow_group_in,
1470 match_criteria_enable,
1471 MLX5_MATCH_MISC_PARAMETERS | match_params);
1472
1473 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1474 misc_parameters.source_port);
1475 }
1476 }
1477
1478 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
esw_vport_tbl_put(struct mlx5_eswitch * esw)1479 static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1480 {
1481 struct mlx5_vport_tbl_attr attr;
1482 struct mlx5_vport *vport;
1483 unsigned long i;
1484
1485 attr.chain = 0;
1486 attr.prio = 1;
1487 mlx5_esw_for_each_vport(esw, i, vport) {
1488 attr.vport = vport->vport;
1489 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1490 mlx5_esw_vporttbl_put(esw, &attr);
1491 }
1492 }
1493
esw_vport_tbl_get(struct mlx5_eswitch * esw)1494 static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1495 {
1496 struct mlx5_vport_tbl_attr attr;
1497 struct mlx5_flow_table *fdb;
1498 struct mlx5_vport *vport;
1499 unsigned long i;
1500
1501 attr.chain = 0;
1502 attr.prio = 1;
1503 mlx5_esw_for_each_vport(esw, i, vport) {
1504 attr.vport = vport->vport;
1505 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1506 fdb = mlx5_esw_vporttbl_get(esw, &attr);
1507 if (IS_ERR(fdb))
1508 goto out;
1509 }
1510 return 0;
1511
1512 out:
1513 esw_vport_tbl_put(esw);
1514 return PTR_ERR(fdb);
1515 }
1516
1517 #define fdb_modify_header_fwd_to_table_supported(esw) \
1518 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
esw_init_chains_offload_flags(struct mlx5_eswitch * esw,u32 * flags)1519 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1520 {
1521 struct mlx5_core_dev *dev = esw->dev;
1522
1523 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1524 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1525
1526 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1527 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1528 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1529 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1530 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1531 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1532 esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1533 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1534 /* Disabled when ttl workaround is needed, e.g
1535 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1536 */
1537 esw_warn(dev,
1538 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1539 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1540 } else {
1541 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1542 esw_info(dev, "Supported tc chains and prios offload\n");
1543 }
1544
1545 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1546 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1547 }
1548
1549 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1550 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1551 {
1552 struct mlx5_core_dev *dev = esw->dev;
1553 struct mlx5_flow_table *nf_ft, *ft;
1554 struct mlx5_chains_attr attr = {};
1555 struct mlx5_fs_chains *chains;
1556 int err;
1557
1558 esw_init_chains_offload_flags(esw, &attr.flags);
1559 attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1560 attr.max_grp_num = esw->params.large_group_num;
1561 attr.default_ft = miss_fdb;
1562 attr.mapping = esw->offloads.reg_c0_obj_pool;
1563
1564 chains = mlx5_chains_create(dev, &attr);
1565 if (IS_ERR(chains)) {
1566 err = PTR_ERR(chains);
1567 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1568 return err;
1569 }
1570 mlx5_chains_print_info(chains);
1571
1572 esw->fdb_table.offloads.esw_chains_priv = chains;
1573
1574 /* Create tc_end_ft which is the always created ft chain */
1575 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1576 1, 0);
1577 if (IS_ERR(nf_ft)) {
1578 err = PTR_ERR(nf_ft);
1579 goto nf_ft_err;
1580 }
1581
1582 /* Always open the root for fast path */
1583 ft = mlx5_chains_get_table(chains, 0, 1, 0);
1584 if (IS_ERR(ft)) {
1585 err = PTR_ERR(ft);
1586 goto level_0_err;
1587 }
1588
1589 /* Open level 1 for split fdb rules now if prios isn't supported */
1590 if (!mlx5_chains_prios_supported(chains)) {
1591 err = esw_vport_tbl_get(esw);
1592 if (err)
1593 goto level_1_err;
1594 }
1595
1596 mlx5_chains_set_end_ft(chains, nf_ft);
1597
1598 return 0;
1599
1600 level_1_err:
1601 mlx5_chains_put_table(chains, 0, 1, 0);
1602 level_0_err:
1603 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1604 nf_ft_err:
1605 mlx5_chains_destroy(chains);
1606 esw->fdb_table.offloads.esw_chains_priv = NULL;
1607
1608 return err;
1609 }
1610
1611 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1612 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1613 {
1614 if (!mlx5_chains_prios_supported(chains))
1615 esw_vport_tbl_put(esw);
1616 mlx5_chains_put_table(chains, 0, 1, 0);
1617 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1618 mlx5_chains_destroy(chains);
1619 }
1620
1621 #else /* CONFIG_MLX5_CLS_ACT */
1622
1623 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1624 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1625 { return 0; }
1626
1627 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1628 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1629 {}
1630
1631 #endif
1632
1633 static int
esw_create_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1634 esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
1635 struct mlx5_flow_table *fdb,
1636 u32 *flow_group_in,
1637 int *ix)
1638 {
1639 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1640 struct mlx5_flow_group *g;
1641 void *match_criteria;
1642 int count, err = 0;
1643
1644 memset(flow_group_in, 0, inlen);
1645
1646 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS);
1647
1648 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1649 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1650
1651 if (!mlx5_eswitch_vport_match_metadata_enabled(esw) &&
1652 MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1653 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1654 misc_parameters.source_eswitch_owner_vhca_id);
1655 MLX5_SET(create_flow_group_in, flow_group_in,
1656 source_eswitch_owner_vhca_id_valid, 1);
1657 }
1658
1659 /* See comment at table_size calculation */
1660 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1661 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1662 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
1663 *ix += count;
1664
1665 g = mlx5_create_flow_group(fdb, flow_group_in);
1666 if (IS_ERR(g)) {
1667 err = PTR_ERR(g);
1668 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1669 goto out;
1670 }
1671 esw->fdb_table.offloads.send_to_vport_grp = g;
1672
1673 out:
1674 return err;
1675 }
1676
1677 static int
esw_create_meta_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1678 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
1679 struct mlx5_flow_table *fdb,
1680 u32 *flow_group_in,
1681 int *ix)
1682 {
1683 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1684 struct mlx5_flow_group *g;
1685 void *match_criteria;
1686 int err = 0;
1687
1688 if (!esw_src_port_rewrite_supported(esw))
1689 return 0;
1690
1691 memset(flow_group_in, 0, inlen);
1692
1693 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1694 MLX5_MATCH_MISC_PARAMETERS_2);
1695
1696 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1697
1698 MLX5_SET(fte_match_param, match_criteria,
1699 misc_parameters_2.metadata_reg_c_0,
1700 mlx5_eswitch_get_vport_metadata_mask());
1701 MLX5_SET(fte_match_param, match_criteria,
1702 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1703
1704 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1705 MLX5_SET(create_flow_group_in, flow_group_in,
1706 end_flow_index, *ix + esw->total_vports - 1);
1707 *ix += esw->total_vports;
1708
1709 g = mlx5_create_flow_group(fdb, flow_group_in);
1710 if (IS_ERR(g)) {
1711 err = PTR_ERR(g);
1712 esw_warn(esw->dev,
1713 "Failed to create send-to-vport meta flow group err(%d)\n", err);
1714 goto send_vport_meta_err;
1715 }
1716 esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1717
1718 return 0;
1719
1720 send_vport_meta_err:
1721 return err;
1722 }
1723
1724 static int
esw_create_peer_esw_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1725 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
1726 struct mlx5_flow_table *fdb,
1727 u32 *flow_group_in,
1728 int *ix)
1729 {
1730 int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1);
1731 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1732 struct mlx5_flow_group *g;
1733 void *match_criteria;
1734 int err = 0;
1735
1736 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
1737 return 0;
1738
1739 memset(flow_group_in, 0, inlen);
1740
1741 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0);
1742
1743 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1744 match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1745 flow_group_in,
1746 match_criteria);
1747
1748 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1749 misc_parameters.source_eswitch_owner_vhca_id);
1750
1751 MLX5_SET(create_flow_group_in, flow_group_in,
1752 source_eswitch_owner_vhca_id_valid, 1);
1753 }
1754
1755 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1756 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1757 *ix + max_peer_ports);
1758 *ix += max_peer_ports + 1;
1759
1760 g = mlx5_create_flow_group(fdb, flow_group_in);
1761 if (IS_ERR(g)) {
1762 err = PTR_ERR(g);
1763 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
1764 goto out;
1765 }
1766 esw->fdb_table.offloads.peer_miss_grp = g;
1767
1768 out:
1769 return err;
1770 }
1771
1772 static int
esw_create_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1773 esw_create_miss_group(struct mlx5_eswitch *esw,
1774 struct mlx5_flow_table *fdb,
1775 u32 *flow_group_in,
1776 int *ix)
1777 {
1778 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1779 struct mlx5_flow_group *g;
1780 void *match_criteria;
1781 int err = 0;
1782 u8 *dmac;
1783
1784 memset(flow_group_in, 0, inlen);
1785
1786 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1787 MLX5_MATCH_OUTER_HEADERS);
1788 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1789 match_criteria);
1790 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1791 outer_headers.dmac_47_16);
1792 dmac[0] = 0x01;
1793
1794 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1795 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1796 *ix + MLX5_ESW_MISS_FLOWS);
1797
1798 g = mlx5_create_flow_group(fdb, flow_group_in);
1799 if (IS_ERR(g)) {
1800 err = PTR_ERR(g);
1801 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
1802 goto miss_err;
1803 }
1804 esw->fdb_table.offloads.miss_grp = g;
1805
1806 err = esw_add_fdb_miss_rule(esw);
1807 if (err)
1808 goto miss_rule_err;
1809
1810 return 0;
1811
1812 miss_rule_err:
1813 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1814 miss_err:
1815 return err;
1816 }
1817
esw_create_offloads_fdb_tables(struct mlx5_eswitch * esw)1818 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1819 {
1820 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1821 struct mlx5_flow_table_attr ft_attr = {};
1822 struct mlx5_core_dev *dev = esw->dev;
1823 struct mlx5_flow_namespace *root_ns;
1824 struct mlx5_flow_table *fdb = NULL;
1825 int table_size, ix = 0, err = 0;
1826 u32 flags = 0, *flow_group_in;
1827
1828 esw_debug(esw->dev, "Create offloads FDB Tables\n");
1829
1830 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1831 if (!flow_group_in)
1832 return -ENOMEM;
1833
1834 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1835 if (!root_ns) {
1836 esw_warn(dev, "Failed to get FDB flow namespace\n");
1837 err = -EOPNOTSUPP;
1838 goto ns_err;
1839 }
1840 esw->fdb_table.offloads.ns = root_ns;
1841 err = mlx5_flow_namespace_set_mode(root_ns,
1842 esw->dev->priv.steering->mode);
1843 if (err) {
1844 esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1845 goto ns_err;
1846 }
1847
1848 /* To be strictly correct:
1849 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1850 * should be:
1851 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1852 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1853 * but as the peer device might not be in switchdev mode it's not
1854 * possible. We use the fact that by default FW sets max vfs and max sfs
1855 * to the same value on both devices. If it needs to be changed in the future note
1856 * the peer miss group should also be created based on the number of
1857 * total vports of the peer (currently is also uses esw->total_vports).
1858 */
1859 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1860 esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS;
1861
1862 /* create the slow path fdb with encap set, so further table instances
1863 * can be created at run time while VFs are probed if the FW allows that.
1864 */
1865 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1866 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1867 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1868
1869 ft_attr.flags = flags;
1870 ft_attr.max_fte = table_size;
1871 ft_attr.prio = FDB_SLOW_PATH;
1872
1873 fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1874 if (IS_ERR(fdb)) {
1875 err = PTR_ERR(fdb);
1876 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1877 goto slow_fdb_err;
1878 }
1879 esw->fdb_table.offloads.slow_fdb = fdb;
1880
1881 /* Create empty TC-miss managed table. This allows plugging in following
1882 * priorities without directly exposing their level 0 table to
1883 * eswitch_offloads and passing it as miss_fdb to following call to
1884 * esw_chains_create().
1885 */
1886 memset(&ft_attr, 0, sizeof(ft_attr));
1887 ft_attr.prio = FDB_TC_MISS;
1888 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr);
1889 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) {
1890 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table);
1891 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1892 goto tc_miss_table_err;
1893 }
1894
1895 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table);
1896 if (err) {
1897 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1898 goto fdb_chains_err;
1899 }
1900
1901 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1902 if (err)
1903 goto send_vport_err;
1904
1905 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1906 if (err)
1907 goto send_vport_meta_err;
1908
1909 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix);
1910 if (err)
1911 goto peer_miss_err;
1912
1913 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix);
1914 if (err)
1915 goto miss_err;
1916
1917 kvfree(flow_group_in);
1918 return 0;
1919
1920 miss_err:
1921 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1922 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1923 peer_miss_err:
1924 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1925 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1926 send_vport_meta_err:
1927 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1928 send_vport_err:
1929 esw_chains_destroy(esw, esw_chains(esw));
1930 fdb_chains_err:
1931 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1932 tc_miss_table_err:
1933 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw));
1934 slow_fdb_err:
1935 /* Holds true only as long as DMFS is the default */
1936 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
1937 ns_err:
1938 kvfree(flow_group_in);
1939 return err;
1940 }
1941
esw_destroy_offloads_fdb_tables(struct mlx5_eswitch * esw)1942 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
1943 {
1944 if (!mlx5_eswitch_get_slow_fdb(esw))
1945 return;
1946
1947 esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
1948 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
1949 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1950 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1951 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1952 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1953 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1954 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1955 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1956
1957 esw_chains_destroy(esw, esw_chains(esw));
1958
1959 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1960 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw));
1961 /* Holds true only as long as DMFS is the default */
1962 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1963 MLX5_FLOW_STEERING_MODE_DMFS);
1964 atomic64_set(&esw->user_count, 0);
1965 }
1966
esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch * esw)1967 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
1968 {
1969 int nvports;
1970
1971 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1972 if (mlx5e_tc_int_port_supported(esw))
1973 nvports += MLX5E_TC_MAX_INT_PORT_NUM;
1974
1975 return nvports;
1976 }
1977
esw_create_offloads_table(struct mlx5_eswitch * esw)1978 static int esw_create_offloads_table(struct mlx5_eswitch *esw)
1979 {
1980 struct mlx5_flow_table_attr ft_attr = {};
1981 struct mlx5_core_dev *dev = esw->dev;
1982 struct mlx5_flow_table *ft_offloads;
1983 struct mlx5_flow_namespace *ns;
1984 int err = 0;
1985
1986 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1987 if (!ns) {
1988 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
1989 return -EOPNOTSUPP;
1990 }
1991
1992 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
1993 MLX5_ESW_FT_OFFLOADS_DROP_RULE;
1994 ft_attr.prio = 1;
1995
1996 ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
1997 if (IS_ERR(ft_offloads)) {
1998 err = PTR_ERR(ft_offloads);
1999 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
2000 return err;
2001 }
2002
2003 esw->offloads.ft_offloads = ft_offloads;
2004 return 0;
2005 }
2006
esw_destroy_offloads_table(struct mlx5_eswitch * esw)2007 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
2008 {
2009 struct mlx5_esw_offload *offloads = &esw->offloads;
2010
2011 mlx5_destroy_flow_table(offloads->ft_offloads);
2012 }
2013
esw_create_vport_rx_group(struct mlx5_eswitch * esw)2014 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
2015 {
2016 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2017 struct mlx5_flow_group *g;
2018 u32 *flow_group_in;
2019 int nvports;
2020 int err = 0;
2021
2022 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
2023 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2024 if (!flow_group_in)
2025 return -ENOMEM;
2026
2027 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0);
2028
2029 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2030 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
2031
2032 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2033
2034 if (IS_ERR(g)) {
2035 err = PTR_ERR(g);
2036 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
2037 goto out;
2038 }
2039
2040 esw->offloads.vport_rx_group = g;
2041 out:
2042 kvfree(flow_group_in);
2043 return err;
2044 }
2045
esw_destroy_vport_rx_group(struct mlx5_eswitch * esw)2046 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
2047 {
2048 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
2049 }
2050
esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch * esw)2051 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
2052 {
2053 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2054 * for the drop rule, which is placed at the end of the table.
2055 * So return the total of vport and int_port as rule index.
2056 */
2057 return esw_get_nr_ft_offloads_steering_src_ports(esw);
2058 }
2059
esw_create_vport_rx_drop_group(struct mlx5_eswitch * esw)2060 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
2061 {
2062 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2063 struct mlx5_flow_group *g;
2064 u32 *flow_group_in;
2065 int flow_index;
2066 int err = 0;
2067
2068 flow_index = esw_create_vport_rx_drop_rule_index(esw);
2069
2070 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2071 if (!flow_group_in)
2072 return -ENOMEM;
2073
2074 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
2075 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
2076
2077 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2078
2079 if (IS_ERR(g)) {
2080 err = PTR_ERR(g);
2081 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
2082 goto out;
2083 }
2084
2085 esw->offloads.vport_rx_drop_group = g;
2086 out:
2087 kvfree(flow_group_in);
2088 return err;
2089 }
2090
esw_destroy_vport_rx_drop_group(struct mlx5_eswitch * esw)2091 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
2092 {
2093 if (esw->offloads.vport_rx_drop_group)
2094 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group);
2095 }
2096
2097 void
mlx5_esw_set_spec_source_port(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_spec * spec)2098 mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw,
2099 u16 vport,
2100 struct mlx5_flow_spec *spec)
2101 {
2102 void *misc;
2103
2104 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2105 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2106 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2107 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2108
2109 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2110 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2111 mlx5_eswitch_get_vport_metadata_mask());
2112
2113 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2114 } else {
2115 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2116 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2117
2118 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2119 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2120
2121 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2122 }
2123 }
2124
2125 struct mlx5_flow_handle *
mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_destination * dest)2126 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
2127 struct mlx5_flow_destination *dest)
2128 {
2129 struct mlx5_flow_act flow_act = {0};
2130 struct mlx5_flow_handle *flow_rule;
2131 struct mlx5_flow_spec *spec;
2132
2133 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2134 if (!spec) {
2135 flow_rule = ERR_PTR(-ENOMEM);
2136 goto out;
2137 }
2138
2139 mlx5_esw_set_spec_source_port(esw, vport, spec);
2140
2141 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2142 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
2143 &flow_act, dest, 1);
2144 if (IS_ERR(flow_rule)) {
2145 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
2146 goto out;
2147 }
2148
2149 out:
2150 kvfree(spec);
2151 return flow_rule;
2152 }
2153
esw_create_vport_rx_drop_rule(struct mlx5_eswitch * esw)2154 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2155 {
2156 struct mlx5_flow_act flow_act = {};
2157 struct mlx5_flow_handle *flow_rule;
2158
2159 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2160 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL,
2161 &flow_act, NULL, 0);
2162 if (IS_ERR(flow_rule)) {
2163 esw_warn(esw->dev,
2164 "fs offloads: Failed to add vport rx drop rule err %ld\n",
2165 PTR_ERR(flow_rule));
2166 return PTR_ERR(flow_rule);
2167 }
2168
2169 esw->offloads.vport_rx_drop_rule = flow_rule;
2170
2171 return 0;
2172 }
2173
esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch * esw)2174 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2175 {
2176 if (esw->offloads.vport_rx_drop_rule)
2177 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule);
2178 }
2179
mlx5_eswitch_inline_mode_get(struct mlx5_eswitch * esw,u8 * mode)2180 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2181 {
2182 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2183 struct mlx5_core_dev *dev = esw->dev;
2184 struct mlx5_vport *vport;
2185 unsigned long i;
2186
2187 if (!MLX5_CAP_GEN(dev, vport_group_manager))
2188 return -EOPNOTSUPP;
2189
2190 if (!mlx5_esw_is_fdb_created(esw))
2191 return -EOPNOTSUPP;
2192
2193 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2194 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2195 mlx5_mode = MLX5_INLINE_MODE_NONE;
2196 goto out;
2197 case MLX5_CAP_INLINE_MODE_L2:
2198 mlx5_mode = MLX5_INLINE_MODE_L2;
2199 goto out;
2200 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2201 goto query_vports;
2202 }
2203
2204 query_vports:
2205 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
2206 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2207 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode);
2208 if (prev_mlx5_mode != mlx5_mode)
2209 return -EINVAL;
2210 prev_mlx5_mode = mlx5_mode;
2211 }
2212
2213 out:
2214 *mode = mlx5_mode;
2215 return 0;
2216 }
2217
esw_destroy_restore_table(struct mlx5_eswitch * esw)2218 static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2219 {
2220 struct mlx5_esw_offload *offloads = &esw->offloads;
2221
2222 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2223 return;
2224
2225 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
2226 mlx5_destroy_flow_group(offloads->restore_group);
2227 mlx5_destroy_flow_table(offloads->ft_offloads_restore);
2228 }
2229
esw_create_restore_table(struct mlx5_eswitch * esw)2230 static int esw_create_restore_table(struct mlx5_eswitch *esw)
2231 {
2232 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2233 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2234 struct mlx5_flow_table_attr ft_attr = {};
2235 struct mlx5_core_dev *dev = esw->dev;
2236 struct mlx5_flow_namespace *ns;
2237 struct mlx5_modify_hdr *mod_hdr;
2238 void *match_criteria, *misc;
2239 struct mlx5_flow_table *ft;
2240 struct mlx5_flow_group *g;
2241 u32 *flow_group_in;
2242 int err = 0;
2243
2244 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2245 return 0;
2246
2247 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2248 if (!ns) {
2249 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2250 return -EOPNOTSUPP;
2251 }
2252
2253 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2254 if (!flow_group_in) {
2255 err = -ENOMEM;
2256 goto out_free;
2257 }
2258
2259 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2260 ft = mlx5_create_flow_table(ns, &ft_attr);
2261 if (IS_ERR(ft)) {
2262 err = PTR_ERR(ft);
2263 esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2264 err);
2265 goto out_free;
2266 }
2267
2268 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2269 match_criteria);
2270 misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2271 misc_parameters_2);
2272
2273 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2274 ESW_REG_C0_USER_DATA_METADATA_MASK);
2275 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2276 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2277 ft_attr.max_fte - 1);
2278 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2279 MLX5_MATCH_MISC_PARAMETERS_2);
2280 g = mlx5_create_flow_group(ft, flow_group_in);
2281 if (IS_ERR(g)) {
2282 err = PTR_ERR(g);
2283 esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2284 err);
2285 goto err_group;
2286 }
2287
2288 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2289 MLX5_SET(copy_action_in, modact, src_field,
2290 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2291 MLX5_SET(copy_action_in, modact, dst_field,
2292 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2293 mod_hdr = mlx5_modify_header_alloc(esw->dev,
2294 MLX5_FLOW_NAMESPACE_KERNEL, 1,
2295 modact);
2296 if (IS_ERR(mod_hdr)) {
2297 err = PTR_ERR(mod_hdr);
2298 esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2299 err);
2300 goto err_mod_hdr;
2301 }
2302
2303 esw->offloads.ft_offloads_restore = ft;
2304 esw->offloads.restore_group = g;
2305 esw->offloads.restore_copy_hdr_id = mod_hdr;
2306
2307 kvfree(flow_group_in);
2308
2309 return 0;
2310
2311 err_mod_hdr:
2312 mlx5_destroy_flow_group(g);
2313 err_group:
2314 mlx5_destroy_flow_table(ft);
2315 out_free:
2316 kvfree(flow_group_in);
2317
2318 return err;
2319 }
2320
esw_offloads_start(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)2321 static int esw_offloads_start(struct mlx5_eswitch *esw,
2322 struct netlink_ext_ack *extack)
2323 {
2324 int err;
2325
2326 esw->mode = MLX5_ESWITCH_OFFLOADS;
2327 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs);
2328 if (err) {
2329 NL_SET_ERR_MSG_MOD(extack,
2330 "Failed setting eswitch to offloads");
2331 esw->mode = MLX5_ESWITCH_LEGACY;
2332 mlx5_rescan_drivers(esw->dev);
2333 return err;
2334 }
2335 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2336 if (mlx5_eswitch_inline_mode_get(esw,
2337 &esw->offloads.inline_mode)) {
2338 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2339 NL_SET_ERR_MSG_MOD(extack,
2340 "Inline mode is different between vports");
2341 }
2342 }
2343 return 0;
2344 }
2345
mlx5_esw_offloads_rep_init(struct mlx5_eswitch * esw,const struct mlx5_vport * vport)2346 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport)
2347 {
2348 struct mlx5_eswitch_rep *rep;
2349 int rep_type;
2350 int err;
2351
2352 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
2353 if (!rep)
2354 return -ENOMEM;
2355
2356 rep->vport = vport->vport;
2357 rep->vport_index = vport->index;
2358 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2359 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
2360
2361 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL);
2362 if (err)
2363 goto insert_err;
2364
2365 return 0;
2366
2367 insert_err:
2368 kfree(rep);
2369 return err;
2370 }
2371
mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep)2372 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2373 struct mlx5_eswitch_rep *rep)
2374 {
2375 xa_erase(&esw->offloads.vport_reps, rep->vport);
2376 kfree(rep);
2377 }
2378
esw_offloads_cleanup_reps(struct mlx5_eswitch * esw)2379 static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2380 {
2381 struct mlx5_eswitch_rep *rep;
2382 unsigned long i;
2383
2384 mlx5_esw_for_each_rep(esw, i, rep)
2385 mlx5_esw_offloads_rep_cleanup(esw, rep);
2386 xa_destroy(&esw->offloads.vport_reps);
2387 }
2388
esw_offloads_init_reps(struct mlx5_eswitch * esw)2389 static int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2390 {
2391 struct mlx5_vport *vport;
2392 unsigned long i;
2393 int err;
2394
2395 xa_init(&esw->offloads.vport_reps);
2396
2397 mlx5_esw_for_each_vport(esw, i, vport) {
2398 err = mlx5_esw_offloads_rep_init(esw, vport);
2399 if (err)
2400 goto err;
2401 }
2402 return 0;
2403
2404 err:
2405 esw_offloads_cleanup_reps(esw);
2406 return err;
2407 }
2408
esw_port_metadata_set(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx)2409 static int esw_port_metadata_set(struct devlink *devlink, u32 id,
2410 struct devlink_param_gset_ctx *ctx)
2411 {
2412 struct mlx5_core_dev *dev = devlink_priv(devlink);
2413 struct mlx5_eswitch *esw = dev->priv.eswitch;
2414 int err = 0;
2415
2416 down_write(&esw->mode_lock);
2417 if (mlx5_esw_is_fdb_created(esw)) {
2418 err = -EBUSY;
2419 goto done;
2420 }
2421 if (!mlx5_esw_vport_match_metadata_supported(esw)) {
2422 err = -EOPNOTSUPP;
2423 goto done;
2424 }
2425 if (ctx->val.vbool)
2426 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2427 else
2428 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
2429 done:
2430 up_write(&esw->mode_lock);
2431 return err;
2432 }
2433
esw_port_metadata_get(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx)2434 static int esw_port_metadata_get(struct devlink *devlink, u32 id,
2435 struct devlink_param_gset_ctx *ctx)
2436 {
2437 struct mlx5_core_dev *dev = devlink_priv(devlink);
2438
2439 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch);
2440 return 0;
2441 }
2442
esw_port_metadata_validate(struct devlink * devlink,u32 id,union devlink_param_value val,struct netlink_ext_ack * extack)2443 static int esw_port_metadata_validate(struct devlink *devlink, u32 id,
2444 union devlink_param_value val,
2445 struct netlink_ext_ack *extack)
2446 {
2447 struct mlx5_core_dev *dev = devlink_priv(devlink);
2448 u8 esw_mode;
2449
2450 esw_mode = mlx5_eswitch_mode(dev);
2451 if (esw_mode == MLX5_ESWITCH_OFFLOADS) {
2452 NL_SET_ERR_MSG_MOD(extack,
2453 "E-Switch must either disabled or non switchdev mode");
2454 return -EBUSY;
2455 }
2456 return 0;
2457 }
2458
2459 static const struct devlink_param esw_devlink_params[] = {
2460 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA,
2461 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL,
2462 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2463 esw_port_metadata_get,
2464 esw_port_metadata_set,
2465 esw_port_metadata_validate),
2466 };
2467
esw_offloads_init(struct mlx5_eswitch * esw)2468 int esw_offloads_init(struct mlx5_eswitch *esw)
2469 {
2470 int err;
2471
2472 err = esw_offloads_init_reps(esw);
2473 if (err)
2474 return err;
2475
2476 if (MLX5_ESWITCH_MANAGER(esw->dev) &&
2477 mlx5_esw_vport_match_metadata_supported(esw))
2478 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2479
2480 err = devl_params_register(priv_to_devlink(esw->dev),
2481 esw_devlink_params,
2482 ARRAY_SIZE(esw_devlink_params));
2483 if (err)
2484 goto err_params;
2485
2486 return 0;
2487
2488 err_params:
2489 esw_offloads_cleanup_reps(esw);
2490 return err;
2491 }
2492
esw_offloads_cleanup(struct mlx5_eswitch * esw)2493 void esw_offloads_cleanup(struct mlx5_eswitch *esw)
2494 {
2495 devl_params_unregister(priv_to_devlink(esw->dev),
2496 esw_devlink_params,
2497 ARRAY_SIZE(esw_devlink_params));
2498 esw_offloads_cleanup_reps(esw);
2499 }
2500
__esw_offloads_load_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2501 static int __esw_offloads_load_rep(struct mlx5_eswitch *esw,
2502 struct mlx5_eswitch_rep *rep, u8 rep_type)
2503 {
2504 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2505 REP_REGISTERED, REP_LOADED) == REP_REGISTERED)
2506 return esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2507
2508 return 0;
2509 }
2510
__esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2511 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2512 struct mlx5_eswitch_rep *rep, u8 rep_type)
2513 {
2514 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2515 REP_LOADED, REP_REGISTERED) == REP_LOADED)
2516 esw->offloads.rep_ops[rep_type]->unload(rep);
2517 }
2518
__unload_reps_all_vport(struct mlx5_eswitch * esw,u8 rep_type)2519 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2520 {
2521 struct mlx5_eswitch_rep *rep;
2522 unsigned long i;
2523
2524 mlx5_esw_for_each_rep(esw, i, rep)
2525 __esw_offloads_unload_rep(esw, rep, rep_type);
2526 }
2527
mlx5_esw_offloads_rep_load(struct mlx5_eswitch * esw,u16 vport_num)2528 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2529 {
2530 struct mlx5_eswitch_rep *rep;
2531 int rep_type;
2532 int err;
2533
2534 rep = mlx5_eswitch_get_rep(esw, vport_num);
2535 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2536 err = __esw_offloads_load_rep(esw, rep, rep_type);
2537 if (err)
2538 goto err_reps;
2539 }
2540
2541 return 0;
2542
2543 err_reps:
2544 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2545 for (--rep_type; rep_type >= 0; rep_type--)
2546 __esw_offloads_unload_rep(esw, rep, rep_type);
2547 return err;
2548 }
2549
mlx5_esw_offloads_rep_unload(struct mlx5_eswitch * esw,u16 vport_num)2550 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2551 {
2552 struct mlx5_eswitch_rep *rep;
2553 int rep_type;
2554
2555 rep = mlx5_eswitch_get_rep(esw, vport_num);
2556 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2557 __esw_offloads_unload_rep(esw, rep, rep_type);
2558 }
2559
mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2560 int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2561 {
2562 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2563 return 0;
2564
2565 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport);
2566 }
2567
mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2568 void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2569 {
2570 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2571 return;
2572
2573 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport);
2574 }
2575
mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport,struct mlx5_devlink_port * dl_port,u32 controller,u32 sfnum)2576 int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
2577 struct mlx5_devlink_port *dl_port,
2578 u32 controller, u32 sfnum)
2579 {
2580 return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum);
2581 }
2582
mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2583 void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2584 {
2585 mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport);
2586 }
2587
mlx5_esw_offloads_load_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2588 int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2589 {
2590 int err;
2591
2592 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2593 return 0;
2594
2595 err = mlx5_esw_offloads_devlink_port_register(esw, vport);
2596 if (err)
2597 return err;
2598
2599 err = mlx5_esw_offloads_rep_load(esw, vport->vport);
2600 if (err)
2601 goto load_err;
2602 return err;
2603
2604 load_err:
2605 mlx5_esw_offloads_devlink_port_unregister(esw, vport);
2606 return err;
2607 }
2608
mlx5_esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2609 void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2610 {
2611 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2612 return;
2613
2614 mlx5_esw_offloads_rep_unload(esw, vport->vport);
2615
2616 mlx5_esw_offloads_devlink_port_unregister(esw, vport);
2617 }
2618
esw_set_slave_root_fdb(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2619 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2620 struct mlx5_core_dev *slave)
2621 {
2622 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2623 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2624 struct mlx5_flow_root_namespace *root;
2625 struct mlx5_flow_namespace *ns;
2626 int err;
2627
2628 MLX5_SET(set_flow_table_root_in, in, opcode,
2629 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2630 MLX5_SET(set_flow_table_root_in, in, table_type,
2631 FS_FT_FDB);
2632
2633 if (master) {
2634 ns = mlx5_get_flow_namespace(master,
2635 MLX5_FLOW_NAMESPACE_FDB);
2636 root = find_root(&ns->node);
2637 mutex_lock(&root->chain_lock);
2638 MLX5_SET(set_flow_table_root_in, in,
2639 table_eswitch_owner_vhca_id_valid, 1);
2640 MLX5_SET(set_flow_table_root_in, in,
2641 table_eswitch_owner_vhca_id,
2642 MLX5_CAP_GEN(master, vhca_id));
2643 MLX5_SET(set_flow_table_root_in, in, table_id,
2644 root->root_ft->id);
2645 } else {
2646 ns = mlx5_get_flow_namespace(slave,
2647 MLX5_FLOW_NAMESPACE_FDB);
2648 root = find_root(&ns->node);
2649 mutex_lock(&root->chain_lock);
2650 MLX5_SET(set_flow_table_root_in, in, table_id,
2651 root->root_ft->id);
2652 }
2653
2654 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2655 mutex_unlock(&root->chain_lock);
2656
2657 return err;
2658 }
2659
__esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,struct mlx5_vport * vport,struct mlx5_flow_table * acl)2660 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2661 struct mlx5_core_dev *slave,
2662 struct mlx5_vport *vport,
2663 struct mlx5_flow_table *acl)
2664 {
2665 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2666 struct mlx5_flow_handle *flow_rule = NULL;
2667 struct mlx5_flow_destination dest = {};
2668 struct mlx5_flow_act flow_act = {};
2669 struct mlx5_flow_spec *spec;
2670 int err = 0;
2671 void *misc;
2672
2673 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2674 if (!spec)
2675 return -ENOMEM;
2676
2677 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2678 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2679 misc_parameters);
2680 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2681 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index);
2682
2683 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2684 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2685 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2686 source_eswitch_owner_vhca_id);
2687
2688 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2689 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2690 dest.vport.num = slave->priv.eswitch->manager_vport;
2691 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2692 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2693
2694 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act,
2695 &dest, 1);
2696 if (IS_ERR(flow_rule)) {
2697 err = PTR_ERR(flow_rule);
2698 } else {
2699 err = xa_insert(&vport->egress.offloads.bounce_rules,
2700 slave_index, flow_rule, GFP_KERNEL);
2701 if (err)
2702 mlx5_del_flow_rules(flow_rule);
2703 }
2704
2705 kvfree(spec);
2706 return err;
2707 }
2708
esw_master_egress_create_resources(struct mlx5_eswitch * esw,struct mlx5_flow_namespace * egress_ns,struct mlx5_vport * vport,size_t count)2709 static int esw_master_egress_create_resources(struct mlx5_eswitch *esw,
2710 struct mlx5_flow_namespace *egress_ns,
2711 struct mlx5_vport *vport, size_t count)
2712 {
2713 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2714 struct mlx5_flow_table_attr ft_attr = {
2715 .max_fte = count, .prio = 0, .level = 0,
2716 };
2717 struct mlx5_flow_table *acl;
2718 struct mlx5_flow_group *g;
2719 void *match_criteria;
2720 u32 *flow_group_in;
2721 int err;
2722
2723 if (vport->egress.acl)
2724 return 0;
2725
2726 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2727 if (!flow_group_in)
2728 return -ENOMEM;
2729
2730 if (vport->vport || mlx5_core_is_ecpf(esw->dev))
2731 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT;
2732
2733 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport);
2734 if (IS_ERR(acl)) {
2735 err = PTR_ERR(acl);
2736 goto out;
2737 }
2738
2739 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2740 match_criteria);
2741 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2742 misc_parameters.source_port);
2743 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2744 misc_parameters.source_eswitch_owner_vhca_id);
2745 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2746 MLX5_MATCH_MISC_PARAMETERS);
2747
2748 MLX5_SET(create_flow_group_in, flow_group_in,
2749 source_eswitch_owner_vhca_id_valid, 1);
2750 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2751 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count);
2752
2753 g = mlx5_create_flow_group(acl, flow_group_in);
2754 if (IS_ERR(g)) {
2755 err = PTR_ERR(g);
2756 goto err_group;
2757 }
2758
2759 vport->egress.acl = acl;
2760 vport->egress.offloads.bounce_grp = g;
2761 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB;
2762 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC);
2763
2764 kvfree(flow_group_in);
2765
2766 return 0;
2767
2768 err_group:
2769 mlx5_destroy_flow_table(acl);
2770 out:
2771 kvfree(flow_group_in);
2772 return err;
2773 }
2774
esw_master_egress_destroy_resources(struct mlx5_vport * vport)2775 static void esw_master_egress_destroy_resources(struct mlx5_vport *vport)
2776 {
2777 if (!xa_empty(&vport->egress.offloads.bounce_rules))
2778 return;
2779 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp);
2780 vport->egress.offloads.bounce_grp = NULL;
2781 mlx5_destroy_flow_table(vport->egress.acl);
2782 vport->egress.acl = NULL;
2783 }
2784
esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,size_t count)2785 static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2786 struct mlx5_core_dev *slave, size_t count)
2787 {
2788 struct mlx5_eswitch *esw = master->priv.eswitch;
2789 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2790 struct mlx5_flow_namespace *egress_ns;
2791 struct mlx5_vport *vport;
2792 int err;
2793
2794 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
2795 if (IS_ERR(vport))
2796 return PTR_ERR(vport);
2797
2798 egress_ns = mlx5_get_flow_vport_acl_namespace(master,
2799 MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2800 vport->index);
2801 if (!egress_ns)
2802 return -EINVAL;
2803
2804 if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB)
2805 return 0;
2806
2807 err = esw_master_egress_create_resources(esw, egress_ns, vport, count);
2808 if (err)
2809 return err;
2810
2811 if (xa_load(&vport->egress.offloads.bounce_rules, slave_index))
2812 return -EINVAL;
2813
2814 err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl);
2815 if (err)
2816 goto err_rule;
2817
2818 return 0;
2819
2820 err_rule:
2821 esw_master_egress_destroy_resources(vport);
2822 return err;
2823 }
2824
esw_unset_master_egress_rule(struct mlx5_core_dev * dev,struct mlx5_core_dev * slave_dev)2825 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev,
2826 struct mlx5_core_dev *slave_dev)
2827 {
2828 struct mlx5_vport *vport;
2829
2830 vport = mlx5_eswitch_get_vport(dev->priv.eswitch,
2831 dev->priv.eswitch->manager_vport);
2832
2833 esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id));
2834
2835 if (xa_empty(&vport->egress.offloads.bounce_rules)) {
2836 esw_acl_egress_ofld_cleanup(vport);
2837 xa_destroy(&vport->egress.offloads.bounce_rules);
2838 }
2839 }
2840
mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw,int max_slaves)2841 int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw,
2842 struct mlx5_eswitch *slave_esw, int max_slaves)
2843 {
2844 int err;
2845
2846 err = esw_set_slave_root_fdb(master_esw->dev,
2847 slave_esw->dev);
2848 if (err)
2849 return err;
2850
2851 err = esw_set_master_egress_rule(master_esw->dev,
2852 slave_esw->dev, max_slaves);
2853 if (err)
2854 goto err_acl;
2855
2856 return err;
2857
2858 err_acl:
2859 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2860 return err;
2861 }
2862
mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2863 void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw,
2864 struct mlx5_eswitch *slave_esw)
2865 {
2866 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2867 esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev);
2868 }
2869
2870 #define ESW_OFFLOADS_DEVCOM_PAIR (0)
2871 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2872
mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2873 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw,
2874 struct mlx5_eswitch *peer_esw)
2875 {
2876 const struct mlx5_eswitch_rep_ops *ops;
2877 struct mlx5_eswitch_rep *rep;
2878 unsigned long i;
2879 u8 rep_type;
2880
2881 mlx5_esw_for_each_rep(esw, i, rep) {
2882 rep_type = NUM_REP_TYPES;
2883 while (rep_type--) {
2884 ops = esw->offloads.rep_ops[rep_type];
2885 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2886 ops->event)
2887 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw);
2888 }
2889 }
2890 }
2891
mlx5_esw_offloads_unpair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2892 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw,
2893 struct mlx5_eswitch *peer_esw)
2894 {
2895 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2896 mlx5e_tc_clean_fdb_peer_flows(esw);
2897 #endif
2898 mlx5_esw_offloads_rep_event_unpair(esw, peer_esw);
2899 esw_del_fdb_peer_miss_rules(esw, peer_esw->dev);
2900 }
2901
mlx5_esw_offloads_pair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2902 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2903 struct mlx5_eswitch *peer_esw)
2904 {
2905 const struct mlx5_eswitch_rep_ops *ops;
2906 struct mlx5_eswitch_rep *rep;
2907 unsigned long i;
2908 u8 rep_type;
2909 int err;
2910
2911 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
2912 if (err)
2913 return err;
2914
2915 mlx5_esw_for_each_rep(esw, i, rep) {
2916 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2917 ops = esw->offloads.rep_ops[rep_type];
2918 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2919 ops->event) {
2920 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
2921 if (err)
2922 goto err_out;
2923 }
2924 }
2925 }
2926
2927 return 0;
2928
2929 err_out:
2930 mlx5_esw_offloads_unpair(esw, peer_esw);
2931 return err;
2932 }
2933
mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,bool pair)2934 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2935 struct mlx5_eswitch *peer_esw,
2936 bool pair)
2937 {
2938 u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
2939 u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
2940 struct mlx5_flow_root_namespace *peer_ns;
2941 struct mlx5_flow_root_namespace *ns;
2942 int err;
2943
2944 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2945 ns = esw->dev->priv.steering->fdb_root_ns;
2946
2947 if (pair) {
2948 err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id);
2949 if (err)
2950 return err;
2951
2952 err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id);
2953 if (err) {
2954 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
2955 return err;
2956 }
2957 } else {
2958 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
2959 mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id);
2960 }
2961
2962 return 0;
2963 }
2964
mlx5_esw_offloads_devcom_event(int event,void * my_data,void * event_data)2965 static int mlx5_esw_offloads_devcom_event(int event,
2966 void *my_data,
2967 void *event_data)
2968 {
2969 struct mlx5_eswitch *esw = my_data;
2970 struct mlx5_eswitch *peer_esw = event_data;
2971 u16 esw_i, peer_esw_i;
2972 bool esw_paired;
2973 int err;
2974
2975 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
2976 esw_i = MLX5_CAP_GEN(esw->dev, vhca_id);
2977 esw_paired = !!xa_load(&esw->paired, peer_esw_i);
2978
2979 switch (event) {
2980 case ESW_OFFLOADS_DEVCOM_PAIR:
2981 if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2982 mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
2983 break;
2984
2985 if (esw_paired)
2986 break;
2987
2988 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
2989 if (err)
2990 goto err_out;
2991
2992 err = mlx5_esw_offloads_pair(esw, peer_esw);
2993 if (err)
2994 goto err_peer;
2995
2996 err = mlx5_esw_offloads_pair(peer_esw, esw);
2997 if (err)
2998 goto err_pair;
2999
3000 err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL);
3001 if (err)
3002 goto err_xa;
3003
3004 err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL);
3005 if (err)
3006 goto err_peer_xa;
3007
3008 esw->num_peers++;
3009 peer_esw->num_peers++;
3010 mlx5_devcom_comp_set_ready(esw->devcom, true);
3011 break;
3012
3013 case ESW_OFFLOADS_DEVCOM_UNPAIR:
3014 if (!esw_paired)
3015 break;
3016
3017 peer_esw->num_peers--;
3018 esw->num_peers--;
3019 if (!esw->num_peers && !peer_esw->num_peers)
3020 mlx5_devcom_comp_set_ready(esw->devcom, false);
3021 xa_erase(&peer_esw->paired, esw_i);
3022 xa_erase(&esw->paired, peer_esw_i);
3023 mlx5_esw_offloads_unpair(peer_esw, esw);
3024 mlx5_esw_offloads_unpair(esw, peer_esw);
3025 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
3026 break;
3027 }
3028
3029 return 0;
3030
3031 err_peer_xa:
3032 xa_erase(&esw->paired, peer_esw_i);
3033 err_xa:
3034 mlx5_esw_offloads_unpair(peer_esw, esw);
3035 err_pair:
3036 mlx5_esw_offloads_unpair(esw, peer_esw);
3037 err_peer:
3038 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
3039 err_out:
3040 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
3041 event, err);
3042 return err;
3043 }
3044
mlx5_esw_offloads_devcom_init(struct mlx5_eswitch * esw,u64 key)3045 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key)
3046 {
3047 int i;
3048
3049 for (i = 0; i < MLX5_MAX_PORTS; i++)
3050 INIT_LIST_HEAD(&esw->offloads.peer_flows[i]);
3051 mutex_init(&esw->offloads.peer_mutex);
3052
3053 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
3054 return;
3055
3056 if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) &&
3057 !mlx5_lag_is_supported(esw->dev))
3058 return;
3059
3060 xa_init(&esw->paired);
3061 esw->num_peers = 0;
3062 esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc,
3063 MLX5_DEVCOM_ESW_OFFLOADS,
3064 key,
3065 mlx5_esw_offloads_devcom_event,
3066 esw);
3067 if (IS_ERR_OR_NULL(esw->devcom))
3068 return;
3069
3070 mlx5_devcom_send_event(esw->devcom,
3071 ESW_OFFLOADS_DEVCOM_PAIR,
3072 ESW_OFFLOADS_DEVCOM_UNPAIR,
3073 esw);
3074 }
3075
mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch * esw)3076 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
3077 {
3078 if (IS_ERR_OR_NULL(esw->devcom))
3079 return;
3080
3081 mlx5_devcom_send_event(esw->devcom,
3082 ESW_OFFLOADS_DEVCOM_UNPAIR,
3083 ESW_OFFLOADS_DEVCOM_UNPAIR,
3084 esw);
3085
3086 mlx5_devcom_unregister_component(esw->devcom);
3087 xa_destroy(&esw->paired);
3088 esw->devcom = NULL;
3089 }
3090
mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch * esw)3091 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw)
3092 {
3093 return mlx5_devcom_comp_is_ready(esw->devcom);
3094 }
3095
mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch * esw)3096 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
3097 {
3098 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
3099 return false;
3100
3101 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
3102 MLX5_FDB_TO_VPORT_REG_C_0))
3103 return false;
3104
3105 return true;
3106 }
3107
3108 #define MLX5_ESW_METADATA_RSVD_UPLINK 1
3109
3110 /* Share the same metadata for uplink's. This is fine because:
3111 * (a) In shared FDB mode (LAG) both uplink's are treated the
3112 * same and tagged with the same metadata.
3113 * (b) In non shared FDB mode, packets from physical port0
3114 * cannot hit eswitch of PF1 and vice versa.
3115 */
mlx5_esw_match_metadata_reserved(struct mlx5_eswitch * esw)3116 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
3117 {
3118 return MLX5_ESW_METADATA_RSVD_UPLINK;
3119 }
3120
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch * esw)3121 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
3122 {
3123 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
3124 /* Reserve 0xf for internal port offload */
3125 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
3126 u32 pf_num;
3127 int id;
3128
3129 /* Only 4 bits of pf_num */
3130 pf_num = mlx5_get_dev_index(esw->dev);
3131 if (pf_num > max_pf_num)
3132 return 0;
3133
3134 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
3135 /* Use only non-zero vport_id (2-4095) for all PF's */
3136 id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
3137 MLX5_ESW_METADATA_RSVD_UPLINK + 1,
3138 vport_end_ida, GFP_KERNEL);
3139 if (id < 0)
3140 return 0;
3141 id = (pf_num << ESW_VPORT_BITS) | id;
3142 return id;
3143 }
3144
mlx5_esw_match_metadata_free(struct mlx5_eswitch * esw,u32 metadata)3145 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
3146 {
3147 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
3148
3149 /* Metadata contains only 12 bits of actual ida id */
3150 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
3151 }
3152
esw_offloads_vport_metadata_setup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3153 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
3154 struct mlx5_vport *vport)
3155 {
3156 if (vport->vport == MLX5_VPORT_UPLINK)
3157 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
3158 else
3159 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
3160
3161 vport->metadata = vport->default_metadata;
3162 return vport->metadata ? 0 : -ENOSPC;
3163 }
3164
esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3165 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
3166 struct mlx5_vport *vport)
3167 {
3168 if (!vport->default_metadata)
3169 return;
3170
3171 if (vport->vport == MLX5_VPORT_UPLINK)
3172 return;
3173
3174 WARN_ON(vport->metadata != vport->default_metadata);
3175 mlx5_esw_match_metadata_free(esw, vport->default_metadata);
3176 }
3177
esw_offloads_metadata_uninit(struct mlx5_eswitch * esw)3178 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
3179 {
3180 struct mlx5_vport *vport;
3181 unsigned long i;
3182
3183 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3184 return;
3185
3186 mlx5_esw_for_each_vport(esw, i, vport)
3187 esw_offloads_vport_metadata_cleanup(esw, vport);
3188 }
3189
esw_offloads_metadata_init(struct mlx5_eswitch * esw)3190 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
3191 {
3192 struct mlx5_vport *vport;
3193 unsigned long i;
3194 int err;
3195
3196 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3197 return 0;
3198
3199 mlx5_esw_for_each_vport(esw, i, vport) {
3200 err = esw_offloads_vport_metadata_setup(esw, vport);
3201 if (err)
3202 goto metadata_err;
3203 }
3204
3205 return 0;
3206
3207 metadata_err:
3208 esw_offloads_metadata_uninit(esw);
3209 return err;
3210 }
3211
3212 int
esw_vport_create_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3213 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
3214 struct mlx5_vport *vport)
3215 {
3216 int err;
3217
3218 err = esw_acl_ingress_ofld_setup(esw, vport);
3219 if (err)
3220 return err;
3221
3222 err = esw_acl_egress_ofld_setup(esw, vport);
3223 if (err)
3224 goto egress_err;
3225
3226 return 0;
3227
3228 egress_err:
3229 esw_acl_ingress_ofld_cleanup(esw, vport);
3230 return err;
3231 }
3232
3233 void
esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3234 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
3235 struct mlx5_vport *vport)
3236 {
3237 esw_acl_egress_ofld_cleanup(vport);
3238 esw_acl_ingress_ofld_cleanup(esw, vport);
3239 }
3240
esw_create_offloads_acl_tables(struct mlx5_eswitch * esw)3241 static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw)
3242 {
3243 struct mlx5_vport *uplink, *manager;
3244 int ret;
3245
3246 uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3247 if (IS_ERR(uplink))
3248 return PTR_ERR(uplink);
3249
3250 ret = esw_vport_create_offloads_acl_tables(esw, uplink);
3251 if (ret)
3252 return ret;
3253
3254 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3255 if (IS_ERR(manager)) {
3256 ret = PTR_ERR(manager);
3257 goto err_manager;
3258 }
3259
3260 ret = esw_vport_create_offloads_acl_tables(esw, manager);
3261 if (ret)
3262 goto err_manager;
3263
3264 return 0;
3265
3266 err_manager:
3267 esw_vport_destroy_offloads_acl_tables(esw, uplink);
3268 return ret;
3269 }
3270
esw_destroy_offloads_acl_tables(struct mlx5_eswitch * esw)3271 static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw)
3272 {
3273 struct mlx5_vport *vport;
3274
3275 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3276 if (!IS_ERR(vport))
3277 esw_vport_destroy_offloads_acl_tables(esw, vport);
3278
3279 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3280 if (!IS_ERR(vport))
3281 esw_vport_destroy_offloads_acl_tables(esw, vport);
3282 }
3283
mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch * esw)3284 int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw)
3285 {
3286 struct mlx5_eswitch_rep *rep;
3287 unsigned long i;
3288 int ret;
3289
3290 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
3291 return 0;
3292
3293 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3294 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
3295 return 0;
3296
3297 ret = __esw_offloads_load_rep(esw, rep, REP_IB);
3298 if (ret)
3299 return ret;
3300
3301 mlx5_esw_for_each_rep(esw, i, rep) {
3302 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
3303 __esw_offloads_load_rep(esw, rep, REP_IB);
3304 }
3305
3306 return 0;
3307 }
3308
esw_offloads_steering_init(struct mlx5_eswitch * esw)3309 static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3310 {
3311 struct mlx5_esw_indir_table *indir;
3312 int err;
3313
3314 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3315 mutex_init(&esw->fdb_table.offloads.vports.lock);
3316 hash_init(esw->fdb_table.offloads.vports.table);
3317 atomic64_set(&esw->user_count, 0);
3318
3319 indir = mlx5_esw_indir_table_init();
3320 if (IS_ERR(indir)) {
3321 err = PTR_ERR(indir);
3322 goto create_indir_err;
3323 }
3324 esw->fdb_table.offloads.indir = indir;
3325
3326 err = esw_create_offloads_acl_tables(esw);
3327 if (err)
3328 goto create_acl_err;
3329
3330 err = esw_create_offloads_table(esw);
3331 if (err)
3332 goto create_offloads_err;
3333
3334 err = esw_create_restore_table(esw);
3335 if (err)
3336 goto create_restore_err;
3337
3338 err = esw_create_offloads_fdb_tables(esw);
3339 if (err)
3340 goto create_fdb_err;
3341
3342 err = esw_create_vport_rx_group(esw);
3343 if (err)
3344 goto create_fg_err;
3345
3346 err = esw_create_vport_rx_drop_group(esw);
3347 if (err)
3348 goto create_rx_drop_fg_err;
3349
3350 err = esw_create_vport_rx_drop_rule(esw);
3351 if (err)
3352 goto create_rx_drop_rule_err;
3353
3354 return 0;
3355
3356 create_rx_drop_rule_err:
3357 esw_destroy_vport_rx_drop_group(esw);
3358 create_rx_drop_fg_err:
3359 esw_destroy_vport_rx_group(esw);
3360 create_fg_err:
3361 esw_destroy_offloads_fdb_tables(esw);
3362 create_fdb_err:
3363 esw_destroy_restore_table(esw);
3364 create_restore_err:
3365 esw_destroy_offloads_table(esw);
3366 create_offloads_err:
3367 esw_destroy_offloads_acl_tables(esw);
3368 create_acl_err:
3369 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3370 create_indir_err:
3371 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3372 return err;
3373 }
3374
esw_offloads_steering_cleanup(struct mlx5_eswitch * esw)3375 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3376 {
3377 esw_destroy_vport_rx_drop_rule(esw);
3378 esw_destroy_vport_rx_drop_group(esw);
3379 esw_destroy_vport_rx_group(esw);
3380 esw_destroy_offloads_fdb_tables(esw);
3381 esw_destroy_restore_table(esw);
3382 esw_destroy_offloads_table(esw);
3383 esw_destroy_offloads_acl_tables(esw);
3384 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3385 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3386 }
3387
3388 static void
esw_vfs_changed_event_handler(struct mlx5_eswitch * esw,const u32 * out)3389 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
3390 {
3391 struct devlink *devlink;
3392 bool host_pf_disabled;
3393 u16 new_num_vfs;
3394
3395 new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3396 host_params_context.host_num_of_vfs);
3397 host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3398 host_params_context.host_pf_disabled);
3399
3400 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3401 return;
3402
3403 devlink = priv_to_devlink(esw->dev);
3404 devl_lock(devlink);
3405 /* Number of VFs can only change from "0 to x" or "x to 0". */
3406 if (esw->esw_funcs.num_vfs > 0) {
3407 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
3408 } else {
3409 int err;
3410
3411 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
3412 MLX5_VPORT_UC_ADDR_CHANGE);
3413 if (err) {
3414 devl_unlock(devlink);
3415 return;
3416 }
3417 }
3418 esw->esw_funcs.num_vfs = new_num_vfs;
3419 devl_unlock(devlink);
3420 }
3421
esw_functions_changed_event_handler(struct work_struct * work)3422 static void esw_functions_changed_event_handler(struct work_struct *work)
3423 {
3424 struct mlx5_host_work *host_work;
3425 struct mlx5_eswitch *esw;
3426 const u32 *out;
3427
3428 host_work = container_of(work, struct mlx5_host_work, work);
3429 esw = host_work->esw;
3430
3431 out = mlx5_esw_query_functions(esw->dev);
3432 if (IS_ERR(out))
3433 goto out;
3434
3435 esw_vfs_changed_event_handler(esw, out);
3436 kvfree(out);
3437 out:
3438 kfree(host_work);
3439 }
3440
mlx5_esw_funcs_changed_handler(struct notifier_block * nb,unsigned long type,void * data)3441 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3442 {
3443 struct mlx5_esw_functions *esw_funcs;
3444 struct mlx5_host_work *host_work;
3445 struct mlx5_eswitch *esw;
3446
3447 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
3448 if (!host_work)
3449 return NOTIFY_DONE;
3450
3451 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3452 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3453
3454 host_work->esw = esw;
3455
3456 INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3457 queue_work(esw->work_queue, &host_work->work);
3458
3459 return NOTIFY_OK;
3460 }
3461
mlx5_esw_host_number_init(struct mlx5_eswitch * esw)3462 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3463 {
3464 const u32 *query_host_out;
3465
3466 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3467 return 0;
3468
3469 query_host_out = mlx5_esw_query_functions(esw->dev);
3470 if (IS_ERR(query_host_out))
3471 return PTR_ERR(query_host_out);
3472
3473 /* Mark non local controller with non zero controller number. */
3474 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3475 host_params_context.host_number);
3476 kvfree(query_host_out);
3477 return 0;
3478 }
3479
mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch * esw,u32 controller)3480 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3481 {
3482 /* Local controller is always valid */
3483 if (controller == 0)
3484 return true;
3485
3486 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3487 return false;
3488
3489 /* External host number starts with zero in device */
3490 return (controller == esw->offloads.host_number + 1);
3491 }
3492
esw_offloads_enable(struct mlx5_eswitch * esw)3493 int esw_offloads_enable(struct mlx5_eswitch *esw)
3494 {
3495 struct mapping_ctx *reg_c0_obj_pool;
3496 struct mlx5_vport *vport;
3497 unsigned long i;
3498 u64 mapping_id;
3499 int err;
3500
3501 mutex_init(&esw->offloads.termtbl_mutex);
3502 mlx5_rdma_enable_roce(esw->dev);
3503
3504 err = mlx5_esw_host_number_init(esw);
3505 if (err)
3506 goto err_metadata;
3507
3508 err = esw_offloads_metadata_init(esw);
3509 if (err)
3510 goto err_metadata;
3511
3512 err = esw_set_passing_vport_metadata(esw, true);
3513 if (err)
3514 goto err_vport_metadata;
3515
3516 mapping_id = mlx5_query_nic_system_image_guid(esw->dev);
3517
3518 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN,
3519 sizeof(struct mlx5_mapped_obj),
3520 ESW_REG_C0_USER_DATA_METADATA_MASK,
3521 true);
3522
3523 if (IS_ERR(reg_c0_obj_pool)) {
3524 err = PTR_ERR(reg_c0_obj_pool);
3525 goto err_pool;
3526 }
3527 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3528
3529 err = esw_offloads_steering_init(esw);
3530 if (err)
3531 goto err_steering_init;
3532
3533 /* Representor will control the vport link state */
3534 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3535 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3536 if (mlx5_core_ec_sriov_enabled(esw->dev))
3537 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs)
3538 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3539
3540 /* Uplink vport rep must load first. */
3541 err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
3542 if (err)
3543 goto err_uplink;
3544
3545 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
3546 if (err)
3547 goto err_vports;
3548
3549 return 0;
3550
3551 err_vports:
3552 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK);
3553 err_uplink:
3554 esw_offloads_steering_cleanup(esw);
3555 err_steering_init:
3556 mapping_destroy(reg_c0_obj_pool);
3557 err_pool:
3558 esw_set_passing_vport_metadata(esw, false);
3559 err_vport_metadata:
3560 esw_offloads_metadata_uninit(esw);
3561 err_metadata:
3562 mlx5_rdma_disable_roce(esw->dev);
3563 mutex_destroy(&esw->offloads.termtbl_mutex);
3564 return err;
3565 }
3566
esw_offloads_stop(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)3567 static int esw_offloads_stop(struct mlx5_eswitch *esw,
3568 struct netlink_ext_ack *extack)
3569 {
3570 int err;
3571
3572 esw->mode = MLX5_ESWITCH_LEGACY;
3573
3574 /* If changing from switchdev to legacy mode without sriov enabled,
3575 * no need to create legacy fdb.
3576 */
3577 if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev))
3578 return 0;
3579
3580 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS);
3581 if (err)
3582 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3583
3584 return err;
3585 }
3586
esw_offloads_disable(struct mlx5_eswitch * esw)3587 void esw_offloads_disable(struct mlx5_eswitch *esw)
3588 {
3589 mlx5_eswitch_disable_pf_vf_vports(esw);
3590 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK);
3591 esw_set_passing_vport_metadata(esw, false);
3592 esw_offloads_steering_cleanup(esw);
3593 mapping_destroy(esw->offloads.reg_c0_obj_pool);
3594 esw_offloads_metadata_uninit(esw);
3595 mlx5_rdma_disable_roce(esw->dev);
3596 mutex_destroy(&esw->offloads.termtbl_mutex);
3597 }
3598
esw_mode_from_devlink(u16 mode,u16 * mlx5_mode)3599 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3600 {
3601 switch (mode) {
3602 case DEVLINK_ESWITCH_MODE_LEGACY:
3603 *mlx5_mode = MLX5_ESWITCH_LEGACY;
3604 break;
3605 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3606 *mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3607 break;
3608 default:
3609 return -EINVAL;
3610 }
3611
3612 return 0;
3613 }
3614
esw_mode_to_devlink(u16 mlx5_mode,u16 * mode)3615 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
3616 {
3617 switch (mlx5_mode) {
3618 case MLX5_ESWITCH_LEGACY:
3619 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
3620 break;
3621 case MLX5_ESWITCH_OFFLOADS:
3622 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3623 break;
3624 default:
3625 return -EINVAL;
3626 }
3627
3628 return 0;
3629 }
3630
esw_inline_mode_from_devlink(u8 mode,u8 * mlx5_mode)3631 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3632 {
3633 switch (mode) {
3634 case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3635 *mlx5_mode = MLX5_INLINE_MODE_NONE;
3636 break;
3637 case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3638 *mlx5_mode = MLX5_INLINE_MODE_L2;
3639 break;
3640 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3641 *mlx5_mode = MLX5_INLINE_MODE_IP;
3642 break;
3643 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3644 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3645 break;
3646 default:
3647 return -EINVAL;
3648 }
3649
3650 return 0;
3651 }
3652
esw_inline_mode_to_devlink(u8 mlx5_mode,u8 * mode)3653 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3654 {
3655 switch (mlx5_mode) {
3656 case MLX5_INLINE_MODE_NONE:
3657 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3658 break;
3659 case MLX5_INLINE_MODE_L2:
3660 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3661 break;
3662 case MLX5_INLINE_MODE_IP:
3663 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3664 break;
3665 case MLX5_INLINE_MODE_TCP_UDP:
3666 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3667 break;
3668 default:
3669 return -EINVAL;
3670 }
3671
3672 return 0;
3673 }
3674
mlx5_eswitch_block_mode(struct mlx5_core_dev * dev)3675 int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev)
3676 {
3677 struct mlx5_eswitch *esw = dev->priv.eswitch;
3678 int err;
3679
3680 if (!mlx5_esw_allowed(esw))
3681 return 0;
3682
3683 /* Take TC into account */
3684 err = mlx5_esw_try_lock(esw);
3685 if (err < 0)
3686 return err;
3687
3688 esw->offloads.num_block_mode++;
3689 mlx5_esw_unlock(esw);
3690 return 0;
3691 }
3692
mlx5_eswitch_unblock_mode(struct mlx5_core_dev * dev)3693 void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev)
3694 {
3695 struct mlx5_eswitch *esw = dev->priv.eswitch;
3696
3697 if (!mlx5_esw_allowed(esw))
3698 return;
3699
3700 down_write(&esw->mode_lock);
3701 esw->offloads.num_block_mode--;
3702 up_write(&esw->mode_lock);
3703 }
3704
mlx5_devlink_eswitch_mode_set(struct devlink * devlink,u16 mode,struct netlink_ext_ack * extack)3705 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
3706 struct netlink_ext_ack *extack)
3707 {
3708 u16 cur_mlx5_mode, mlx5_mode = 0;
3709 struct mlx5_eswitch *esw;
3710 int err = 0;
3711
3712 esw = mlx5_devlink_eswitch_get(devlink);
3713 if (IS_ERR(esw))
3714 return PTR_ERR(esw);
3715
3716 if (esw_mode_from_devlink(mode, &mlx5_mode))
3717 return -EINVAL;
3718
3719 mlx5_lag_disable_change(esw->dev);
3720 err = mlx5_esw_try_lock(esw);
3721 if (err < 0) {
3722 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
3723 goto enable_lag;
3724 }
3725 cur_mlx5_mode = err;
3726 err = 0;
3727
3728 if (cur_mlx5_mode == mlx5_mode)
3729 goto unlock;
3730
3731 if (esw->offloads.num_block_mode) {
3732 NL_SET_ERR_MSG_MOD(extack,
3733 "Can't change eswitch mode when IPsec SA and/or policies are configured");
3734 err = -EOPNOTSUPP;
3735 goto unlock;
3736 }
3737
3738 esw->eswitch_operation_in_progress = true;
3739 up_write(&esw->mode_lock);
3740
3741 mlx5_eswitch_disable_locked(esw);
3742 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
3743 if (mlx5_devlink_trap_get_num_active(esw->dev)) {
3744 NL_SET_ERR_MSG_MOD(extack,
3745 "Can't change mode while devlink traps are active");
3746 err = -EOPNOTSUPP;
3747 goto skip;
3748 }
3749 err = esw_offloads_start(esw, extack);
3750 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
3751 err = esw_offloads_stop(esw, extack);
3752 mlx5_rescan_drivers(esw->dev);
3753 } else {
3754 err = -EINVAL;
3755 }
3756
3757 skip:
3758 down_write(&esw->mode_lock);
3759 esw->eswitch_operation_in_progress = false;
3760 unlock:
3761 mlx5_esw_unlock(esw);
3762 enable_lag:
3763 mlx5_lag_enable_change(esw->dev);
3764 return err;
3765 }
3766
mlx5_devlink_eswitch_mode_get(struct devlink * devlink,u16 * mode)3767 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3768 {
3769 struct mlx5_eswitch *esw;
3770
3771 esw = mlx5_devlink_eswitch_get(devlink);
3772 if (IS_ERR(esw))
3773 return PTR_ERR(esw);
3774
3775 return esw_mode_to_devlink(esw->mode, mode);
3776 }
3777
mlx5_esw_vports_inline_set(struct mlx5_eswitch * esw,u8 mlx5_mode,struct netlink_ext_ack * extack)3778 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
3779 struct netlink_ext_ack *extack)
3780 {
3781 struct mlx5_core_dev *dev = esw->dev;
3782 struct mlx5_vport *vport;
3783 u16 err_vport_num = 0;
3784 unsigned long i;
3785 int err = 0;
3786
3787 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3788 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3789 if (err) {
3790 err_vport_num = vport->vport;
3791 NL_SET_ERR_MSG_MOD(extack,
3792 "Failed to set min inline on vport");
3793 goto revert_inline_mode;
3794 }
3795 }
3796 if (mlx5_core_ec_sriov_enabled(esw->dev)) {
3797 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
3798 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3799 if (err) {
3800 err_vport_num = vport->vport;
3801 NL_SET_ERR_MSG_MOD(extack,
3802 "Failed to set min inline on vport");
3803 goto revert_ec_vf_inline_mode;
3804 }
3805 }
3806 }
3807 return 0;
3808
3809 revert_ec_vf_inline_mode:
3810 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
3811 if (vport->vport == err_vport_num)
3812 break;
3813 mlx5_modify_nic_vport_min_inline(dev,
3814 vport->vport,
3815 esw->offloads.inline_mode);
3816 }
3817 revert_inline_mode:
3818 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3819 if (vport->vport == err_vport_num)
3820 break;
3821 mlx5_modify_nic_vport_min_inline(dev,
3822 vport->vport,
3823 esw->offloads.inline_mode);
3824 }
3825 return err;
3826 }
3827
mlx5_devlink_eswitch_inline_mode_set(struct devlink * devlink,u8 mode,struct netlink_ext_ack * extack)3828 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
3829 struct netlink_ext_ack *extack)
3830 {
3831 struct mlx5_core_dev *dev = devlink_priv(devlink);
3832 struct mlx5_eswitch *esw;
3833 u8 mlx5_mode;
3834 int err;
3835
3836 esw = mlx5_devlink_eswitch_get(devlink);
3837 if (IS_ERR(esw))
3838 return PTR_ERR(esw);
3839
3840 down_write(&esw->mode_lock);
3841
3842 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
3843 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
3844 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
3845 err = 0;
3846 goto out;
3847 }
3848
3849 fallthrough;
3850 case MLX5_CAP_INLINE_MODE_L2:
3851 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
3852 err = -EOPNOTSUPP;
3853 goto out;
3854 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3855 break;
3856 }
3857
3858 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3859 NL_SET_ERR_MSG_MOD(extack,
3860 "Can't set inline mode when flows are configured");
3861 err = -EOPNOTSUPP;
3862 goto out;
3863 }
3864
3865 err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
3866 if (err)
3867 goto out;
3868
3869 esw->eswitch_operation_in_progress = true;
3870 up_write(&esw->mode_lock);
3871
3872 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
3873 if (!err)
3874 esw->offloads.inline_mode = mlx5_mode;
3875
3876 down_write(&esw->mode_lock);
3877 esw->eswitch_operation_in_progress = false;
3878 up_write(&esw->mode_lock);
3879 return 0;
3880
3881 out:
3882 up_write(&esw->mode_lock);
3883 return err;
3884 }
3885
mlx5_devlink_eswitch_inline_mode_get(struct devlink * devlink,u8 * mode)3886 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
3887 {
3888 struct mlx5_eswitch *esw;
3889
3890 esw = mlx5_devlink_eswitch_get(devlink);
3891 if (IS_ERR(esw))
3892 return PTR_ERR(esw);
3893
3894 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
3895 }
3896
mlx5_eswitch_block_encap(struct mlx5_core_dev * dev)3897 bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev)
3898 {
3899 struct mlx5_eswitch *esw = dev->priv.eswitch;
3900
3901 if (!mlx5_esw_allowed(esw))
3902 return true;
3903
3904 down_write(&esw->mode_lock);
3905 if (esw->mode != MLX5_ESWITCH_LEGACY &&
3906 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
3907 up_write(&esw->mode_lock);
3908 return false;
3909 }
3910
3911 esw->offloads.num_block_encap++;
3912 up_write(&esw->mode_lock);
3913 return true;
3914 }
3915
mlx5_eswitch_unblock_encap(struct mlx5_core_dev * dev)3916 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev)
3917 {
3918 struct mlx5_eswitch *esw = dev->priv.eswitch;
3919
3920 if (!mlx5_esw_allowed(esw))
3921 return;
3922
3923 down_write(&esw->mode_lock);
3924 esw->offloads.num_block_encap--;
3925 up_write(&esw->mode_lock);
3926 }
3927
mlx5_devlink_eswitch_encap_mode_set(struct devlink * devlink,enum devlink_eswitch_encap_mode encap,struct netlink_ext_ack * extack)3928 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
3929 enum devlink_eswitch_encap_mode encap,
3930 struct netlink_ext_ack *extack)
3931 {
3932 struct mlx5_core_dev *dev = devlink_priv(devlink);
3933 struct mlx5_eswitch *esw;
3934 int err = 0;
3935
3936 esw = mlx5_devlink_eswitch_get(devlink);
3937 if (IS_ERR(esw))
3938 return PTR_ERR(esw);
3939
3940 down_write(&esw->mode_lock);
3941
3942 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
3943 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
3944 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
3945 err = -EOPNOTSUPP;
3946 goto unlock;
3947 }
3948
3949 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
3950 err = -EOPNOTSUPP;
3951 goto unlock;
3952 }
3953
3954 if (esw->mode == MLX5_ESWITCH_LEGACY) {
3955 esw->offloads.encap = encap;
3956 goto unlock;
3957 }
3958
3959 if (esw->offloads.encap == encap)
3960 goto unlock;
3961
3962 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3963 NL_SET_ERR_MSG_MOD(extack,
3964 "Can't set encapsulation when flows are configured");
3965 err = -EOPNOTSUPP;
3966 goto unlock;
3967 }
3968
3969 if (esw->offloads.num_block_encap) {
3970 NL_SET_ERR_MSG_MOD(extack,
3971 "Can't set encapsulation when IPsec SA and/or policies are configured");
3972 err = -EOPNOTSUPP;
3973 goto unlock;
3974 }
3975
3976 esw->eswitch_operation_in_progress = true;
3977 up_write(&esw->mode_lock);
3978
3979 esw_destroy_offloads_fdb_tables(esw);
3980
3981 esw->offloads.encap = encap;
3982
3983 err = esw_create_offloads_fdb_tables(esw);
3984
3985 if (err) {
3986 NL_SET_ERR_MSG_MOD(extack,
3987 "Failed re-creating fast FDB table");
3988 esw->offloads.encap = !encap;
3989 (void)esw_create_offloads_fdb_tables(esw);
3990 }
3991
3992 down_write(&esw->mode_lock);
3993 esw->eswitch_operation_in_progress = false;
3994
3995 unlock:
3996 up_write(&esw->mode_lock);
3997 return err;
3998 }
3999
mlx5_devlink_eswitch_encap_mode_get(struct devlink * devlink,enum devlink_eswitch_encap_mode * encap)4000 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
4001 enum devlink_eswitch_encap_mode *encap)
4002 {
4003 struct mlx5_eswitch *esw;
4004
4005 esw = mlx5_devlink_eswitch_get(devlink);
4006 if (IS_ERR(esw))
4007 return PTR_ERR(esw);
4008
4009 *encap = esw->offloads.encap;
4010 return 0;
4011 }
4012
4013 static bool
mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch * esw,u16 vport_num)4014 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
4015 {
4016 /* Currently, only ECPF based device has representor for host PF. */
4017 if (vport_num == MLX5_VPORT_PF &&
4018 !mlx5_core_is_ecpf_esw_manager(esw->dev))
4019 return false;
4020
4021 if (vport_num == MLX5_VPORT_ECPF &&
4022 !mlx5_ecpf_vport_exists(esw->dev))
4023 return false;
4024
4025 return true;
4026 }
4027
mlx5_eswitch_register_vport_reps(struct mlx5_eswitch * esw,const struct mlx5_eswitch_rep_ops * ops,u8 rep_type)4028 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
4029 const struct mlx5_eswitch_rep_ops *ops,
4030 u8 rep_type)
4031 {
4032 struct mlx5_eswitch_rep_data *rep_data;
4033 struct mlx5_eswitch_rep *rep;
4034 unsigned long i;
4035
4036 esw->offloads.rep_ops[rep_type] = ops;
4037 mlx5_esw_for_each_rep(esw, i, rep) {
4038 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
4039 rep->esw = esw;
4040 rep_data = &rep->rep_data[rep_type];
4041 atomic_set(&rep_data->state, REP_REGISTERED);
4042 }
4043 }
4044 }
4045 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
4046
mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch * esw,u8 rep_type)4047 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
4048 {
4049 struct mlx5_eswitch_rep *rep;
4050 unsigned long i;
4051
4052 if (esw->mode == MLX5_ESWITCH_OFFLOADS)
4053 __unload_reps_all_vport(esw, rep_type);
4054
4055 mlx5_esw_for_each_rep(esw, i, rep)
4056 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
4057 }
4058 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
4059
mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch * esw,u8 rep_type)4060 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
4061 {
4062 struct mlx5_eswitch_rep *rep;
4063
4064 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
4065 return rep->rep_data[rep_type].priv;
4066 }
4067
mlx5_eswitch_get_proto_dev(struct mlx5_eswitch * esw,u16 vport,u8 rep_type)4068 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
4069 u16 vport,
4070 u8 rep_type)
4071 {
4072 struct mlx5_eswitch_rep *rep;
4073
4074 rep = mlx5_eswitch_get_rep(esw, vport);
4075
4076 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
4077 esw->offloads.rep_ops[rep_type]->get_proto_dev)
4078 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
4079 return NULL;
4080 }
4081 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
4082
mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch * esw,u8 rep_type)4083 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
4084 {
4085 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
4086 }
4087 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
4088
mlx5_eswitch_vport_rep(struct mlx5_eswitch * esw,u16 vport)4089 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
4090 u16 vport)
4091 {
4092 return mlx5_eswitch_get_rep(esw, vport);
4093 }
4094 EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
4095
mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch * esw)4096 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
4097 {
4098 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
4099 }
4100 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
4101
mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch * esw)4102 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
4103 {
4104 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
4105 }
4106 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
4107
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch * esw,u16 vport_num)4108 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
4109 u16 vport_num)
4110 {
4111 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4112
4113 if (WARN_ON_ONCE(IS_ERR(vport)))
4114 return 0;
4115
4116 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
4117 }
4118 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
4119
mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch * esw,u16 vport_num,u16 * vhca_id)4120 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
4121 {
4122 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4123 void *query_ctx;
4124 void *hca_caps;
4125 int err;
4126
4127 *vhca_id = 0;
4128
4129 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4130 if (!query_ctx)
4131 return -ENOMEM;
4132
4133 err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx);
4134 if (err)
4135 goto out_free;
4136
4137 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4138 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
4139
4140 out_free:
4141 kfree(query_ctx);
4142 return err;
4143 }
4144
mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch * esw,u16 vport_num)4145 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
4146 {
4147 u16 *old_entry, *vhca_map_entry, vhca_id;
4148 int err;
4149
4150 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
4151 if (err) {
4152 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
4153 vport_num, err);
4154 return err;
4155 }
4156
4157 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
4158 if (!vhca_map_entry)
4159 return -ENOMEM;
4160
4161 *vhca_map_entry = vport_num;
4162 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
4163 if (xa_is_err(old_entry)) {
4164 kfree(vhca_map_entry);
4165 return xa_err(old_entry);
4166 }
4167 kfree(old_entry);
4168 return 0;
4169 }
4170
mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch * esw,u16 vport_num)4171 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
4172 {
4173 u16 *vhca_map_entry, vhca_id;
4174 int err;
4175
4176 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
4177 if (err)
4178 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
4179 vport_num, err);
4180
4181 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id);
4182 kfree(vhca_map_entry);
4183 }
4184
mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch * esw,u16 vhca_id,u16 * vport_num)4185 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
4186 {
4187 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
4188
4189 if (!res)
4190 return -ENOENT;
4191
4192 *vport_num = *res;
4193 return 0;
4194 }
4195
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch * esw,u16 vport_num)4196 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
4197 u16 vport_num)
4198 {
4199 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4200
4201 if (WARN_ON_ONCE(IS_ERR(vport)))
4202 return 0;
4203
4204 return vport->metadata;
4205 }
4206 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
4207
mlx5_devlink_port_fn_hw_addr_get(struct devlink_port * port,u8 * hw_addr,int * hw_addr_len,struct netlink_ext_ack * extack)4208 int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port,
4209 u8 *hw_addr, int *hw_addr_len,
4210 struct netlink_ext_ack *extack)
4211 {
4212 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4213 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4214
4215 mutex_lock(&esw->state_lock);
4216 ether_addr_copy(hw_addr, vport->info.mac);
4217 *hw_addr_len = ETH_ALEN;
4218 mutex_unlock(&esw->state_lock);
4219 return 0;
4220 }
4221
mlx5_devlink_port_fn_hw_addr_set(struct devlink_port * port,const u8 * hw_addr,int hw_addr_len,struct netlink_ext_ack * extack)4222 int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port,
4223 const u8 *hw_addr, int hw_addr_len,
4224 struct netlink_ext_ack *extack)
4225 {
4226 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4227 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4228
4229 return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr);
4230 }
4231
mlx5_devlink_port_fn_migratable_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4232 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled,
4233 struct netlink_ext_ack *extack)
4234 {
4235 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4236 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4237
4238 if (!MLX5_CAP_GEN(esw->dev, migration)) {
4239 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4240 return -EOPNOTSUPP;
4241 }
4242
4243 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4244 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4245 return -EOPNOTSUPP;
4246 }
4247
4248 mutex_lock(&esw->state_lock);
4249 *is_enabled = vport->info.mig_enabled;
4250 mutex_unlock(&esw->state_lock);
4251 return 0;
4252 }
4253
mlx5_devlink_port_fn_migratable_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4254 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable,
4255 struct netlink_ext_ack *extack)
4256 {
4257 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4258 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4259 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4260 void *query_ctx;
4261 void *hca_caps;
4262 int err;
4263
4264 if (!MLX5_CAP_GEN(esw->dev, migration)) {
4265 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4266 return -EOPNOTSUPP;
4267 }
4268
4269 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4270 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4271 return -EOPNOTSUPP;
4272 }
4273
4274 mutex_lock(&esw->state_lock);
4275
4276 if (vport->info.mig_enabled == enable) {
4277 err = 0;
4278 goto out;
4279 }
4280
4281 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4282 if (!query_ctx) {
4283 err = -ENOMEM;
4284 goto out;
4285 }
4286
4287 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx,
4288 MLX5_CAP_GENERAL_2);
4289 if (err) {
4290 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4291 goto out_free;
4292 }
4293
4294 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4295 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable);
4296
4297 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport,
4298 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2);
4299 if (err) {
4300 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap");
4301 goto out_free;
4302 }
4303
4304 vport->info.mig_enabled = enable;
4305
4306 out_free:
4307 kfree(query_ctx);
4308 out:
4309 mutex_unlock(&esw->state_lock);
4310 return err;
4311 }
4312
mlx5_devlink_port_fn_roce_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4313 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled,
4314 struct netlink_ext_ack *extack)
4315 {
4316 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4317 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4318
4319 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4320 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4321 return -EOPNOTSUPP;
4322 }
4323
4324 mutex_lock(&esw->state_lock);
4325 *is_enabled = vport->info.roce_enabled;
4326 mutex_unlock(&esw->state_lock);
4327 return 0;
4328 }
4329
mlx5_devlink_port_fn_roce_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4330 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable,
4331 struct netlink_ext_ack *extack)
4332 {
4333 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4334 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4335 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4336 u16 vport_num = vport->vport;
4337 void *query_ctx;
4338 void *hca_caps;
4339 int err;
4340
4341 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4342 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4343 return -EOPNOTSUPP;
4344 }
4345
4346 mutex_lock(&esw->state_lock);
4347
4348 if (vport->info.roce_enabled == enable) {
4349 err = 0;
4350 goto out;
4351 }
4352
4353 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4354 if (!query_ctx) {
4355 err = -ENOMEM;
4356 goto out;
4357 }
4358
4359 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx,
4360 MLX5_CAP_GENERAL);
4361 if (err) {
4362 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4363 goto out_free;
4364 }
4365
4366 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4367 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable);
4368
4369 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num,
4370 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
4371 if (err) {
4372 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap");
4373 goto out_free;
4374 }
4375
4376 vport->info.roce_enabled = enable;
4377
4378 out_free:
4379 kfree(query_ctx);
4380 out:
4381 mutex_unlock(&esw->state_lock);
4382 return err;
4383 }
4384
4385 int
mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_esw_flow_attr * esw_attr,int attr_idx)4386 mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule,
4387 struct mlx5_esw_flow_attr *esw_attr, int attr_idx)
4388 {
4389 struct mlx5_flow_destination new_dest = {};
4390 struct mlx5_flow_destination old_dest = {};
4391
4392 if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
4393 return 0;
4394
4395 esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false);
4396 esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false);
4397
4398 return mlx5_modify_rule_destination(rule, &new_dest, &old_dest);
4399 }
4400
4401 #ifdef CONFIG_XFRM_OFFLOAD
mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4402 int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled,
4403 struct netlink_ext_ack *extack)
4404 {
4405 struct mlx5_eswitch *esw;
4406 struct mlx5_vport *vport;
4407 int err = 0;
4408
4409 esw = mlx5_devlink_eswitch_get(port->devlink);
4410 if (IS_ERR(esw))
4411 return PTR_ERR(esw);
4412
4413 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) {
4414 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto");
4415 return -EOPNOTSUPP;
4416 }
4417
4418 vport = mlx5_devlink_port_vport_get(port);
4419
4420 mutex_lock(&esw->state_lock);
4421 if (!vport->enabled) {
4422 err = -EOPNOTSUPP;
4423 goto unlock;
4424 }
4425
4426 *is_enabled = vport->info.ipsec_crypto_enabled;
4427 unlock:
4428 mutex_unlock(&esw->state_lock);
4429 return err;
4430 }
4431
mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4432 int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable,
4433 struct netlink_ext_ack *extack)
4434 {
4435 struct mlx5_eswitch *esw;
4436 struct mlx5_vport *vport;
4437 u16 vport_num;
4438 int err;
4439
4440 esw = mlx5_devlink_eswitch_get(port->devlink);
4441 if (IS_ERR(esw))
4442 return PTR_ERR(esw);
4443
4444 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4445 err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num);
4446 if (err) {
4447 NL_SET_ERR_MSG_MOD(extack,
4448 "Device doesn't support IPsec crypto");
4449 return err;
4450 }
4451
4452 vport = mlx5_devlink_port_vport_get(port);
4453
4454 mutex_lock(&esw->state_lock);
4455 if (!vport->enabled) {
4456 err = -EOPNOTSUPP;
4457 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4458 goto unlock;
4459 }
4460
4461 if (vport->info.ipsec_crypto_enabled == enable)
4462 goto unlock;
4463
4464 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4465 err = -EBUSY;
4466 goto unlock;
4467 }
4468
4469 err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable);
4470 if (err) {
4471 NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto");
4472 goto unlock;
4473 }
4474
4475 vport->info.ipsec_crypto_enabled = enable;
4476 if (enable)
4477 esw->enabled_ipsec_vf_count++;
4478 else
4479 esw->enabled_ipsec_vf_count--;
4480 unlock:
4481 mutex_unlock(&esw->state_lock);
4482 return err;
4483 }
4484
mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4485 int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled,
4486 struct netlink_ext_ack *extack)
4487 {
4488 struct mlx5_eswitch *esw;
4489 struct mlx5_vport *vport;
4490 int err = 0;
4491
4492 esw = mlx5_devlink_eswitch_get(port->devlink);
4493 if (IS_ERR(esw))
4494 return PTR_ERR(esw);
4495
4496 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) {
4497 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet");
4498 return -EOPNOTSUPP;
4499 }
4500
4501 vport = mlx5_devlink_port_vport_get(port);
4502
4503 mutex_lock(&esw->state_lock);
4504 if (!vport->enabled) {
4505 err = -EOPNOTSUPP;
4506 goto unlock;
4507 }
4508
4509 *is_enabled = vport->info.ipsec_packet_enabled;
4510 unlock:
4511 mutex_unlock(&esw->state_lock);
4512 return err;
4513 }
4514
mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4515 int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port,
4516 bool enable,
4517 struct netlink_ext_ack *extack)
4518 {
4519 struct mlx5_eswitch *esw;
4520 struct mlx5_vport *vport;
4521 u16 vport_num;
4522 int err;
4523
4524 esw = mlx5_devlink_eswitch_get(port->devlink);
4525 if (IS_ERR(esw))
4526 return PTR_ERR(esw);
4527
4528 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4529 err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num);
4530 if (err) {
4531 NL_SET_ERR_MSG_MOD(extack,
4532 "Device doesn't support IPsec packet mode");
4533 return err;
4534 }
4535
4536 vport = mlx5_devlink_port_vport_get(port);
4537 mutex_lock(&esw->state_lock);
4538 if (!vport->enabled) {
4539 err = -EOPNOTSUPP;
4540 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4541 goto unlock;
4542 }
4543
4544 if (vport->info.ipsec_packet_enabled == enable)
4545 goto unlock;
4546
4547 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4548 err = -EBUSY;
4549 goto unlock;
4550 }
4551
4552 err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable);
4553 if (err) {
4554 NL_SET_ERR_MSG_MOD(extack,
4555 "Failed to set IPsec packet mode");
4556 goto unlock;
4557 }
4558
4559 vport->info.ipsec_packet_enabled = enable;
4560 if (enable)
4561 esw->enabled_ipsec_vf_count++;
4562 else
4563 esw->enabled_ipsec_vf_count--;
4564 unlock:
4565 mutex_unlock(&esw->state_lock);
4566 return err;
4567 }
4568 #endif /* CONFIG_XFRM_OFFLOAD */
4569