1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include <linux/mlx5/fs.h> 5 #include "eswitch.h" 6 #include "en_tc.h" 7 #include "fs_core.h" 8 9 struct mlx5_termtbl_handle { 10 struct hlist_node termtbl_hlist; 11 12 struct mlx5_flow_table *termtbl; 13 struct mlx5_flow_act flow_act; 14 struct mlx5_flow_destination dest; 15 16 struct mlx5_flow_handle *rule; 17 int ref_count; 18 }; 19 20 static u32 21 mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act, 22 struct mlx5_flow_destination *dest) 23 { 24 u32 hash; 25 26 hash = jhash_1word(flow_act->action, 0); 27 hash = jhash((const void *)&flow_act->vlan, 28 sizeof(flow_act->vlan), hash); 29 hash = jhash((const void *)&dest->vport.num, 30 sizeof(dest->vport.num), hash); 31 hash = jhash((const void *)&dest->vport.vhca_id, 32 sizeof(dest->vport.num), hash); 33 if (flow_act->pkt_reformat) 34 hash = jhash(flow_act->pkt_reformat, 35 sizeof(*flow_act->pkt_reformat), 36 hash); 37 return hash; 38 } 39 40 static int 41 mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1, 42 struct mlx5_flow_destination *dest1, 43 struct mlx5_flow_act *flow_act2, 44 struct mlx5_flow_destination *dest2) 45 { 46 int ret; 47 48 ret = flow_act1->action != flow_act2->action || 49 dest1->vport.num != dest2->vport.num || 50 dest1->vport.vhca_id != dest2->vport.vhca_id || 51 memcmp(&flow_act1->vlan, &flow_act2->vlan, 52 sizeof(flow_act1->vlan)); 53 if (ret) 54 return ret; 55 56 if (flow_act1->pkt_reformat && flow_act2->pkt_reformat) 57 return memcmp(flow_act1->pkt_reformat, flow_act2->pkt_reformat, 58 sizeof(*flow_act1->pkt_reformat)); 59 60 return !(flow_act1->pkt_reformat == flow_act2->pkt_reformat); 61 } 62 63 static int 64 mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev, 65 struct mlx5_termtbl_handle *tt, 66 struct mlx5_flow_act *flow_act) 67 { 68 struct mlx5_flow_table_attr ft_attr = {}; 69 struct mlx5_flow_namespace *root_ns; 70 int err, err2; 71 72 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 73 if (!root_ns) { 74 esw_warn(dev, "Failed to get FDB flow namespace\n"); 75 return -EOPNOTSUPP; 76 } 77 78 /* As this is the terminating action then the termination table is the 79 * same prio as the slow path 80 */ 81 ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED | 82 MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 83 ft_attr.prio = FDB_TC_OFFLOAD; 84 ft_attr.max_fte = 1; 85 ft_attr.level = 1; 86 ft_attr.autogroup.max_num_groups = 1; 87 tt->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr); 88 if (IS_ERR(tt->termtbl)) { 89 err = PTR_ERR(tt->termtbl); 90 esw_warn(dev, "Failed to create termination table, err %pe\n", tt->termtbl); 91 return err; 92 } 93 94 tt->rule = mlx5_add_flow_rules(tt->termtbl, NULL, flow_act, 95 &tt->dest, 1); 96 if (IS_ERR(tt->rule)) { 97 err = PTR_ERR(tt->rule); 98 esw_warn(dev, "Failed to create termination table rule, err %pe\n", tt->rule); 99 goto add_flow_err; 100 } 101 return 0; 102 103 add_flow_err: 104 err2 = mlx5_destroy_flow_table(tt->termtbl); 105 if (err2) 106 esw_warn(dev, "Failed to destroy termination table, err %d\n", err2); 107 108 return err; 109 } 110 111 static struct mlx5_termtbl_handle * 112 mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch *esw, 113 struct mlx5_flow_act *flow_act, 114 struct mlx5_flow_destination *dest, 115 struct mlx5_esw_flow_attr *attr) 116 { 117 struct mlx5_termtbl_handle *tt; 118 bool found = false; 119 u32 hash_key; 120 int err; 121 122 mutex_lock(&esw->offloads.termtbl_mutex); 123 hash_key = mlx5_eswitch_termtbl_hash(flow_act, dest); 124 hash_for_each_possible(esw->offloads.termtbl_tbl, tt, 125 termtbl_hlist, hash_key) { 126 if (!mlx5_eswitch_termtbl_cmp(&tt->flow_act, &tt->dest, 127 flow_act, dest)) { 128 found = true; 129 break; 130 } 131 } 132 if (found) 133 goto tt_add_ref; 134 135 tt = kzalloc(sizeof(*tt), GFP_KERNEL); 136 if (!tt) { 137 err = -ENOMEM; 138 goto tt_create_err; 139 } 140 141 tt->dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 142 tt->dest.vport.num = dest->vport.num; 143 tt->dest.vport.vhca_id = dest->vport.vhca_id; 144 tt->dest.vport.flags = dest->vport.flags; 145 memcpy(&tt->flow_act, flow_act, sizeof(*flow_act)); 146 147 err = mlx5_eswitch_termtbl_create(esw->dev, tt, flow_act); 148 if (err) 149 goto tt_create_err; 150 151 hash_add(esw->offloads.termtbl_tbl, &tt->termtbl_hlist, hash_key); 152 tt_add_ref: 153 tt->ref_count++; 154 mutex_unlock(&esw->offloads.termtbl_mutex); 155 return tt; 156 tt_create_err: 157 kfree(tt); 158 mutex_unlock(&esw->offloads.termtbl_mutex); 159 return ERR_PTR(err); 160 } 161 162 void 163 mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw, 164 struct mlx5_termtbl_handle *tt) 165 { 166 mutex_lock(&esw->offloads.termtbl_mutex); 167 if (--tt->ref_count == 0) 168 hash_del(&tt->termtbl_hlist); 169 mutex_unlock(&esw->offloads.termtbl_mutex); 170 171 if (!tt->ref_count) { 172 mlx5_del_flow_rules(tt->rule); 173 mlx5_destroy_flow_table(tt->termtbl); 174 kfree(tt); 175 } 176 } 177 178 static void 179 mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src, 180 struct mlx5_flow_act *dst) 181 { 182 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 183 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH; 184 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH; 185 memcpy(&dst->vlan[0], &src->vlan[0], sizeof(src->vlan[0])); 186 memset(&src->vlan[0], 0, sizeof(src->vlan[0])); 187 188 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 189 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2; 190 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2; 191 memcpy(&dst->vlan[1], &src->vlan[1], sizeof(src->vlan[1])); 192 memset(&src->vlan[1], 0, sizeof(src->vlan[1])); 193 } 194 } 195 } 196 197 static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw, 198 const struct mlx5_flow_spec *spec) 199 { 200 u16 port_mask, port_value; 201 202 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) 203 return spec->flow_context.flow_source == 204 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 205 206 port_mask = MLX5_GET(fte_match_param, spec->match_criteria, 207 misc_parameters.source_port); 208 port_value = MLX5_GET(fte_match_param, spec->match_value, 209 misc_parameters.source_port); 210 return (port_mask & port_value) == MLX5_VPORT_UPLINK; 211 } 212 213 static bool 214 mlx5_eswitch_is_push_vlan_no_cap(struct mlx5_eswitch *esw, 215 struct mlx5_flow_act *flow_act) 216 { 217 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH && 218 !(mlx5_fs_get_capabilities(esw->dev, MLX5_FLOW_NAMESPACE_FDB) & 219 MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX)) 220 return true; 221 222 return false; 223 } 224 225 bool 226 mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw, 227 struct mlx5_flow_attr *attr, 228 struct mlx5_flow_act *flow_act, 229 struct mlx5_flow_spec *spec) 230 { 231 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 232 int i; 233 234 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) || 235 !MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level) || 236 mlx5e_tc_attr_flags_skip(attr->flags) || 237 (!mlx5_eswitch_offload_is_uplink_port(esw, spec) && !esw_attr->int_port)) 238 return false; 239 240 if (mlx5_eswitch_is_push_vlan_no_cap(esw, flow_act)) 241 return true; 242 243 /* hairpin */ 244 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 245 if (!esw_attr->dest_int_port && esw_attr->dests[i].rep && 246 esw_attr->dests[i].rep->vport == MLX5_VPORT_UPLINK) 247 return true; 248 249 return false; 250 } 251 252 struct mlx5_flow_handle * 253 mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw, 254 struct mlx5_flow_table *fdb, 255 struct mlx5_flow_spec *spec, 256 struct mlx5_esw_flow_attr *attr, 257 struct mlx5_flow_act *flow_act, 258 struct mlx5_flow_destination *dest, 259 int num_dest) 260 { 261 struct mlx5_flow_act term_tbl_act = {}; 262 struct mlx5_flow_handle *rule = NULL; 263 bool term_table_created = false; 264 bool is_push_vlan_on_rx; 265 int num_vport_dests = 0; 266 int i, curr_dest; 267 268 is_push_vlan_on_rx = mlx5_eswitch_is_push_vlan_no_cap(esw, flow_act); 269 mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act); 270 term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 271 272 for (i = 0; i < num_dest; i++) { 273 struct mlx5_termtbl_handle *tt; 274 bool hairpin = false; 275 276 /* only vport destinations can be terminated */ 277 if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT) 278 continue; 279 280 if (attr->dests[num_vport_dests].rep && 281 attr->dests[num_vport_dests].rep->vport == MLX5_VPORT_UPLINK) 282 hairpin = true; 283 284 if (!is_push_vlan_on_rx && !hairpin) { 285 num_vport_dests++; 286 continue; 287 } 288 289 if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) { 290 term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 291 term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat; 292 } else { 293 term_tbl_act.action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 294 term_tbl_act.pkt_reformat = NULL; 295 } 296 297 /* get the terminating table for the action list */ 298 tt = mlx5_eswitch_termtbl_get_create(esw, &term_tbl_act, 299 &dest[i], attr); 300 if (IS_ERR(tt)) { 301 esw_warn(esw->dev, "Failed to get termination table, err %pe\n", tt); 302 goto revert_changes; 303 } 304 attr->dests[num_vport_dests].termtbl = tt; 305 num_vport_dests++; 306 307 /* link the destination with the termination table */ 308 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 309 dest[i].ft = tt->termtbl; 310 term_table_created = true; 311 } 312 313 /* at least one destination should reference a termination table */ 314 if (!term_table_created) 315 goto revert_changes; 316 317 /* create the FTE */ 318 flow_act->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 319 flow_act->pkt_reformat = NULL; 320 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 321 rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest); 322 if (IS_ERR(rule)) 323 goto revert_changes; 324 325 goto out; 326 327 revert_changes: 328 /* revert the changes that were made to the original flow_act 329 * and fall-back to the original rule actions 330 */ 331 mlx5_eswitch_termtbl_actions_move(&term_tbl_act, flow_act); 332 333 for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) { 334 struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl; 335 336 if (!tt) 337 continue; 338 339 attr->dests[curr_dest].termtbl = NULL; 340 341 /* search for the destination associated with the 342 * current term table 343 */ 344 for (i = 0; i < num_dest; i++) { 345 if (dest[i].ft != tt->termtbl) 346 continue; 347 348 memset(&dest[i], 0, sizeof(dest[i])); 349 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 350 dest[i].vport.num = tt->dest.vport.num; 351 dest[i].vport.vhca_id = tt->dest.vport.vhca_id; 352 mlx5_eswitch_termtbl_put(esw, tt); 353 break; 354 } 355 } 356 rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest); 357 out: 358 return rule; 359 } 360