1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include <linux/mlx5/fs.h> 5 #include "eswitch.h" 6 #include "en_tc.h" 7 #include "fs_core.h" 8 9 struct mlx5_termtbl_handle { 10 struct hlist_node termtbl_hlist; 11 12 struct mlx5_flow_table *termtbl; 13 struct mlx5_flow_act flow_act; 14 struct mlx5_flow_destination dest; 15 16 struct mlx5_flow_handle *rule; 17 int ref_count; 18 }; 19 20 static u32 21 mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act, 22 struct mlx5_flow_destination *dest) 23 { 24 u32 hash; 25 26 hash = jhash_1word(flow_act->action, 0); 27 hash = jhash((const void *)&flow_act->vlan, 28 sizeof(flow_act->vlan), hash); 29 hash = jhash((const void *)&dest->vport.num, 30 sizeof(dest->vport.num), hash); 31 hash = jhash((const void *)&dest->vport.vhca_id, 32 sizeof(dest->vport.num), hash); 33 if (dest->vport.pkt_reformat) 34 hash = jhash(dest->vport.pkt_reformat, 35 sizeof(*dest->vport.pkt_reformat), 36 hash); 37 return hash; 38 } 39 40 static int 41 mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1, 42 struct mlx5_flow_destination *dest1, 43 struct mlx5_flow_act *flow_act2, 44 struct mlx5_flow_destination *dest2) 45 { 46 int ret; 47 48 ret = flow_act1->action != flow_act2->action || 49 dest1->vport.num != dest2->vport.num || 50 dest1->vport.vhca_id != dest2->vport.vhca_id || 51 memcmp(&flow_act1->vlan, &flow_act2->vlan, 52 sizeof(flow_act1->vlan)); 53 if (ret) 54 return ret; 55 56 return dest1->vport.pkt_reformat && dest2->vport.pkt_reformat ? 57 memcmp(dest1->vport.pkt_reformat, dest2->vport.pkt_reformat, 58 sizeof(*dest1->vport.pkt_reformat)) : 0; 59 } 60 61 static int 62 mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev, 63 struct mlx5_termtbl_handle *tt, 64 struct mlx5_flow_act *flow_act) 65 { 66 struct mlx5_flow_table_attr ft_attr = {}; 67 struct mlx5_flow_namespace *root_ns; 68 int err; 69 70 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 71 if (!root_ns) { 72 esw_warn(dev, "Failed to get FDB flow namespace\n"); 73 return -EOPNOTSUPP; 74 } 75 76 /* As this is the terminating action then the termination table is the 77 * same prio as the slow path 78 */ 79 ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | 80 MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 81 ft_attr.prio = FDB_SLOW_PATH; 82 ft_attr.max_fte = 1; 83 ft_attr.autogroup.max_num_groups = 1; 84 tt->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr); 85 if (IS_ERR(tt->termtbl)) { 86 esw_warn(dev, "Failed to create termination table (error %d)\n", 87 IS_ERR(tt->termtbl)); 88 return -EOPNOTSUPP; 89 } 90 91 tt->rule = mlx5_add_flow_rules(tt->termtbl, NULL, flow_act, 92 &tt->dest, 1); 93 if (IS_ERR(tt->rule)) { 94 esw_warn(dev, "Failed to create termination table rule (error %d)\n", 95 IS_ERR(tt->rule)); 96 goto add_flow_err; 97 } 98 return 0; 99 100 add_flow_err: 101 err = mlx5_destroy_flow_table(tt->termtbl); 102 if (err) 103 esw_warn(dev, "Failed to destroy termination table\n"); 104 105 return -EOPNOTSUPP; 106 } 107 108 static struct mlx5_termtbl_handle * 109 mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch *esw, 110 struct mlx5_flow_act *flow_act, 111 struct mlx5_flow_destination *dest, 112 struct mlx5_esw_flow_attr *attr) 113 { 114 struct mlx5_termtbl_handle *tt; 115 bool found = false; 116 u32 hash_key; 117 int err; 118 119 mutex_lock(&esw->offloads.termtbl_mutex); 120 hash_key = mlx5_eswitch_termtbl_hash(flow_act, dest); 121 hash_for_each_possible(esw->offloads.termtbl_tbl, tt, 122 termtbl_hlist, hash_key) { 123 if (!mlx5_eswitch_termtbl_cmp(&tt->flow_act, &tt->dest, 124 flow_act, dest)) { 125 found = true; 126 break; 127 } 128 } 129 if (found) 130 goto tt_add_ref; 131 132 tt = kzalloc(sizeof(*tt), GFP_KERNEL); 133 if (!tt) { 134 err = -ENOMEM; 135 goto tt_create_err; 136 } 137 138 tt->dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 139 tt->dest.vport.num = dest->vport.num; 140 tt->dest.vport.vhca_id = dest->vport.vhca_id; 141 tt->dest.vport.flags = dest->vport.flags; 142 memcpy(&tt->flow_act, flow_act, sizeof(*flow_act)); 143 144 err = mlx5_eswitch_termtbl_create(esw->dev, tt, flow_act); 145 if (err) 146 goto tt_create_err; 147 148 hash_add(esw->offloads.termtbl_tbl, &tt->termtbl_hlist, hash_key); 149 tt_add_ref: 150 tt->ref_count++; 151 mutex_unlock(&esw->offloads.termtbl_mutex); 152 return tt; 153 tt_create_err: 154 kfree(tt); 155 mutex_unlock(&esw->offloads.termtbl_mutex); 156 return ERR_PTR(err); 157 } 158 159 void 160 mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw, 161 struct mlx5_termtbl_handle *tt) 162 { 163 mutex_lock(&esw->offloads.termtbl_mutex); 164 if (--tt->ref_count == 0) 165 hash_del(&tt->termtbl_hlist); 166 mutex_unlock(&esw->offloads.termtbl_mutex); 167 168 if (!tt->ref_count) { 169 mlx5_del_flow_rules(tt->rule); 170 mlx5_destroy_flow_table(tt->termtbl); 171 kfree(tt); 172 } 173 } 174 175 static bool mlx5_eswitch_termtbl_is_encap_reformat(struct mlx5_pkt_reformat *rt) 176 { 177 switch (rt->reformat_type) { 178 case MLX5_REFORMAT_TYPE_L2_TO_VXLAN: 179 case MLX5_REFORMAT_TYPE_L2_TO_NVGRE: 180 case MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 181 case MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 182 return true; 183 default: 184 return false; 185 } 186 } 187 188 static void 189 mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src, 190 struct mlx5_flow_act *dst) 191 { 192 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 193 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH; 194 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH; 195 memcpy(&dst->vlan[0], &src->vlan[0], sizeof(src->vlan[0])); 196 memset(&src->vlan[0], 0, sizeof(src->vlan[0])); 197 198 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 199 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2; 200 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2; 201 memcpy(&dst->vlan[1], &src->vlan[1], sizeof(src->vlan[1])); 202 memset(&src->vlan[1], 0, sizeof(src->vlan[1])); 203 } 204 } 205 206 if (src->action & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT && 207 mlx5_eswitch_termtbl_is_encap_reformat(src->pkt_reformat)) { 208 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 209 dst->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 210 dst->pkt_reformat = src->pkt_reformat; 211 src->pkt_reformat = NULL; 212 } 213 } 214 215 static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw, 216 const struct mlx5_flow_spec *spec) 217 { 218 u16 port_mask, port_value; 219 220 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) 221 return spec->flow_context.flow_source == 222 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 223 224 port_mask = MLX5_GET(fte_match_param, spec->match_criteria, 225 misc_parameters.source_port); 226 port_value = MLX5_GET(fte_match_param, spec->match_value, 227 misc_parameters.source_port); 228 return (port_mask & port_value) == MLX5_VPORT_UPLINK; 229 } 230 231 bool 232 mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw, 233 struct mlx5_flow_attr *attr, 234 struct mlx5_flow_act *flow_act, 235 struct mlx5_flow_spec *spec) 236 { 237 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 238 int i; 239 240 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) || 241 attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH || 242 !mlx5_eswitch_offload_is_uplink_port(esw, spec)) 243 return false; 244 245 /* push vlan on RX */ 246 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) 247 return true; 248 249 /* hairpin */ 250 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 251 if (esw_attr->dests[i].rep->vport == MLX5_VPORT_UPLINK) 252 return true; 253 254 return false; 255 } 256 257 struct mlx5_flow_handle * 258 mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw, 259 struct mlx5_flow_table *fdb, 260 struct mlx5_flow_spec *spec, 261 struct mlx5_esw_flow_attr *attr, 262 struct mlx5_flow_act *flow_act, 263 struct mlx5_flow_destination *dest, 264 int num_dest) 265 { 266 struct mlx5_flow_act term_tbl_act = {}; 267 struct mlx5_flow_handle *rule = NULL; 268 bool term_table_created = false; 269 int num_vport_dests = 0; 270 int i, curr_dest; 271 272 mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act); 273 term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 274 275 for (i = 0; i < num_dest; i++) { 276 struct mlx5_termtbl_handle *tt; 277 278 /* only vport destinations can be terminated */ 279 if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT) 280 continue; 281 282 /* get the terminating table for the action list */ 283 tt = mlx5_eswitch_termtbl_get_create(esw, &term_tbl_act, 284 &dest[i], attr); 285 if (IS_ERR(tt)) { 286 esw_warn(esw->dev, "Failed to get termination table (error %d)\n", 287 IS_ERR(tt)); 288 goto revert_changes; 289 } 290 attr->dests[num_vport_dests].termtbl = tt; 291 num_vport_dests++; 292 293 /* link the destination with the termination table */ 294 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 295 dest[i].ft = tt->termtbl; 296 term_table_created = true; 297 } 298 299 /* at least one destination should reference a termination table */ 300 if (!term_table_created) 301 goto revert_changes; 302 303 /* create the FTE */ 304 rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest); 305 if (IS_ERR(rule)) 306 goto revert_changes; 307 308 goto out; 309 310 revert_changes: 311 /* revert the changes that were made to the original flow_act 312 * and fall-back to the original rule actions 313 */ 314 mlx5_eswitch_termtbl_actions_move(&term_tbl_act, flow_act); 315 316 for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) { 317 struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl; 318 319 /* search for the destination associated with the 320 * current term table 321 */ 322 for (i = 0; i < num_dest; i++) { 323 if (dest[i].ft != tt->termtbl) 324 continue; 325 326 memset(&dest[i], 0, sizeof(dest[i])); 327 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 328 dest[i].vport.num = tt->dest.vport.num; 329 dest[i].vport.vhca_id = tt->dest.vport.vhca_id; 330 mlx5_eswitch_termtbl_put(esw, tt); 331 break; 332 } 333 } 334 rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest); 335 out: 336 return rule; 337 } 338