1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2021 Mellanox Technologies. */ 3 4 #include <linux/skbuff.h> 5 #include <net/psample.h> 6 #include "en/mapping.h" 7 #include "en/tc/post_act.h" 8 #include "sample.h" 9 #include "eswitch.h" 10 #include "en_tc.h" 11 #include "fs_core.h" 12 13 #define MLX5_ESW_VPORT_TBL_SIZE_SAMPLE (64 * 1024) 14 15 static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_sample_ns = { 16 .max_fte = MLX5_ESW_VPORT_TBL_SIZE_SAMPLE, 17 .max_num_groups = 0, /* default num of groups */ 18 .flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | MLX5_FLOW_TABLE_TUNNEL_EN_DECAP, 19 }; 20 21 struct mlx5e_tc_psample { 22 struct mlx5_eswitch *esw; 23 struct mlx5_flow_table *termtbl; 24 struct mlx5_flow_handle *termtbl_rule; 25 DECLARE_HASHTABLE(hashtbl, 8); 26 struct mutex ht_lock; /* protect hashtbl */ 27 DECLARE_HASHTABLE(restore_hashtbl, 8); 28 struct mutex restore_lock; /* protect restore_hashtbl */ 29 struct mlx5e_post_act *post_act; 30 }; 31 32 struct mlx5e_sampler { 33 struct hlist_node hlist; 34 u32 sampler_id; 35 u32 sample_ratio; 36 u32 sample_table_id; 37 u32 default_table_id; 38 int count; 39 }; 40 41 struct mlx5e_sample_flow { 42 struct mlx5e_sampler *sampler; 43 struct mlx5e_sample_restore *restore; 44 struct mlx5_flow_attr *pre_attr; 45 struct mlx5_flow_handle *pre_rule; 46 struct mlx5_flow_attr *post_attr; 47 struct mlx5_flow_handle *post_rule; 48 struct mlx5e_post_act_handle *post_act_handle; 49 }; 50 51 struct mlx5e_sample_restore { 52 struct hlist_node hlist; 53 struct mlx5_modify_hdr *modify_hdr; 54 struct mlx5_flow_handle *rule; 55 struct mlx5e_post_act_handle *post_act_handle; 56 u32 obj_id; 57 int count; 58 }; 59 60 static int 61 sampler_termtbl_create(struct mlx5e_tc_psample *tc_psample) 62 { 63 struct mlx5_eswitch *esw = tc_psample->esw; 64 struct mlx5_flow_table_attr ft_attr = {}; 65 struct mlx5_flow_destination dest = {}; 66 struct mlx5_core_dev *dev = esw->dev; 67 struct mlx5_flow_namespace *root_ns; 68 struct mlx5_flow_act act = {}; 69 int err; 70 71 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, termination_table)) { 72 mlx5_core_warn(dev, "termination table is not supported\n"); 73 return -EOPNOTSUPP; 74 } 75 76 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 77 if (!root_ns) { 78 mlx5_core_warn(dev, "failed to get FDB flow namespace\n"); 79 return -EOPNOTSUPP; 80 } 81 82 ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED; 83 ft_attr.autogroup.max_num_groups = 1; 84 ft_attr.prio = FDB_SLOW_PATH; 85 ft_attr.max_fte = 1; 86 ft_attr.level = 1; 87 tc_psample->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr); 88 if (IS_ERR(tc_psample->termtbl)) { 89 err = PTR_ERR(tc_psample->termtbl); 90 mlx5_core_warn(dev, "failed to create termtbl, err: %d\n", err); 91 return err; 92 } 93 94 act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 95 dest.vport.num = esw->manager_vport; 96 tc_psample->termtbl_rule = mlx5_add_flow_rules(tc_psample->termtbl, NULL, &act, &dest, 1); 97 if (IS_ERR(tc_psample->termtbl_rule)) { 98 err = PTR_ERR(tc_psample->termtbl_rule); 99 mlx5_core_warn(dev, "failed to create termtbl rule, err: %d\n", err); 100 mlx5_destroy_flow_table(tc_psample->termtbl); 101 return err; 102 } 103 104 return 0; 105 } 106 107 static void 108 sampler_termtbl_destroy(struct mlx5e_tc_psample *tc_psample) 109 { 110 mlx5_del_flow_rules(tc_psample->termtbl_rule); 111 mlx5_destroy_flow_table(tc_psample->termtbl); 112 } 113 114 static int 115 sampler_obj_create(struct mlx5_core_dev *mdev, struct mlx5e_sampler *sampler) 116 { 117 u32 in[MLX5_ST_SZ_DW(create_sampler_obj_in)] = {}; 118 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 119 u64 general_obj_types; 120 void *obj; 121 int err; 122 123 general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); 124 if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_SAMPLER)) 125 return -EOPNOTSUPP; 126 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, ignore_flow_level)) 127 return -EOPNOTSUPP; 128 129 obj = MLX5_ADDR_OF(create_sampler_obj_in, in, sampler_object); 130 MLX5_SET(sampler_obj, obj, table_type, FS_FT_FDB); 131 MLX5_SET(sampler_obj, obj, ignore_flow_level, 1); 132 MLX5_SET(sampler_obj, obj, level, 1); 133 MLX5_SET(sampler_obj, obj, sample_ratio, sampler->sample_ratio); 134 MLX5_SET(sampler_obj, obj, sample_table_id, sampler->sample_table_id); 135 MLX5_SET(sampler_obj, obj, default_table_id, sampler->default_table_id); 136 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 137 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_SAMPLER); 138 139 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 140 if (!err) 141 sampler->sampler_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 142 143 return err; 144 } 145 146 static void 147 sampler_obj_destroy(struct mlx5_core_dev *mdev, u32 sampler_id) 148 { 149 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; 150 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 151 152 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 153 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_SAMPLER); 154 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sampler_id); 155 156 mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 157 } 158 159 static u32 160 sampler_hash(u32 sample_ratio, u32 default_table_id) 161 { 162 return jhash_2words(sample_ratio, default_table_id, 0); 163 } 164 165 static int 166 sampler_cmp(u32 sample_ratio1, u32 default_table_id1, u32 sample_ratio2, u32 default_table_id2) 167 { 168 return sample_ratio1 != sample_ratio2 || default_table_id1 != default_table_id2; 169 } 170 171 static struct mlx5e_sampler * 172 sampler_get(struct mlx5e_tc_psample *tc_psample, u32 sample_ratio, u32 default_table_id) 173 { 174 struct mlx5e_sampler *sampler; 175 u32 hash_key; 176 int err; 177 178 mutex_lock(&tc_psample->ht_lock); 179 hash_key = sampler_hash(sample_ratio, default_table_id); 180 hash_for_each_possible(tc_psample->hashtbl, sampler, hlist, hash_key) 181 if (!sampler_cmp(sampler->sample_ratio, sampler->default_table_id, 182 sample_ratio, default_table_id)) 183 goto add_ref; 184 185 sampler = kzalloc(sizeof(*sampler), GFP_KERNEL); 186 if (!sampler) { 187 err = -ENOMEM; 188 goto err_alloc; 189 } 190 191 sampler->sample_table_id = tc_psample->termtbl->id; 192 sampler->default_table_id = default_table_id; 193 sampler->sample_ratio = sample_ratio; 194 195 err = sampler_obj_create(tc_psample->esw->dev, sampler); 196 if (err) 197 goto err_create; 198 199 hash_add(tc_psample->hashtbl, &sampler->hlist, hash_key); 200 201 add_ref: 202 sampler->count++; 203 mutex_unlock(&tc_psample->ht_lock); 204 return sampler; 205 206 err_create: 207 kfree(sampler); 208 err_alloc: 209 mutex_unlock(&tc_psample->ht_lock); 210 return ERR_PTR(err); 211 } 212 213 static void 214 sampler_put(struct mlx5e_tc_psample *tc_psample, struct mlx5e_sampler *sampler) 215 { 216 mutex_lock(&tc_psample->ht_lock); 217 if (--sampler->count == 0) { 218 hash_del(&sampler->hlist); 219 sampler_obj_destroy(tc_psample->esw->dev, sampler->sampler_id); 220 kfree(sampler); 221 } 222 mutex_unlock(&tc_psample->ht_lock); 223 } 224 225 /* obj_id is used to restore the sample parameters. 226 * Set fte_id in original flow table, then match it in the default table. 227 * Only set it for NICs can preserve reg_c or decap action. For other cases, 228 * use the same match in the default table. 229 * Use one header rewrite for both obj_id and fte_id. 230 */ 231 static struct mlx5_modify_hdr * 232 sample_modify_hdr_get(struct mlx5_core_dev *mdev, u32 obj_id, 233 struct mlx5e_post_act_handle *handle) 234 { 235 struct mlx5e_tc_mod_hdr_acts mod_acts = {}; 236 struct mlx5_modify_hdr *modify_hdr; 237 int err; 238 239 err = mlx5e_tc_match_to_reg_set(mdev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB, 240 CHAIN_TO_REG, obj_id); 241 if (err) 242 goto err_set_regc0; 243 244 if (handle) { 245 err = mlx5e_tc_post_act_set_handle(mdev, handle, &mod_acts); 246 if (err) 247 goto err_post_act; 248 } 249 250 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_FDB, 251 mod_acts.num_actions, 252 mod_acts.actions); 253 if (IS_ERR(modify_hdr)) { 254 err = PTR_ERR(modify_hdr); 255 goto err_modify_hdr; 256 } 257 258 dealloc_mod_hdr_actions(&mod_acts); 259 return modify_hdr; 260 261 err_modify_hdr: 262 err_post_act: 263 dealloc_mod_hdr_actions(&mod_acts); 264 err_set_regc0: 265 return ERR_PTR(err); 266 } 267 268 static u32 269 restore_hash(u32 obj_id, struct mlx5e_post_act_handle *post_act_handle) 270 { 271 return jhash_2words(obj_id, hash32_ptr(post_act_handle), 0); 272 } 273 274 static bool 275 restore_equal(struct mlx5e_sample_restore *restore, u32 obj_id, 276 struct mlx5e_post_act_handle *post_act_handle) 277 { 278 return restore->obj_id == obj_id && restore->post_act_handle == post_act_handle; 279 } 280 281 static struct mlx5e_sample_restore * 282 sample_restore_get(struct mlx5e_tc_psample *tc_psample, u32 obj_id, 283 struct mlx5e_post_act_handle *post_act_handle) 284 { 285 struct mlx5_eswitch *esw = tc_psample->esw; 286 struct mlx5_core_dev *mdev = esw->dev; 287 struct mlx5e_sample_restore *restore; 288 struct mlx5_modify_hdr *modify_hdr; 289 u32 hash_key; 290 int err; 291 292 mutex_lock(&tc_psample->restore_lock); 293 hash_key = restore_hash(obj_id, post_act_handle); 294 hash_for_each_possible(tc_psample->restore_hashtbl, restore, hlist, hash_key) 295 if (restore_equal(restore, obj_id, post_act_handle)) 296 goto add_ref; 297 298 restore = kzalloc(sizeof(*restore), GFP_KERNEL); 299 if (!restore) { 300 err = -ENOMEM; 301 goto err_alloc; 302 } 303 restore->obj_id = obj_id; 304 restore->post_act_handle = post_act_handle; 305 306 modify_hdr = sample_modify_hdr_get(mdev, obj_id, post_act_handle); 307 if (IS_ERR(modify_hdr)) { 308 err = PTR_ERR(modify_hdr); 309 goto err_modify_hdr; 310 } 311 restore->modify_hdr = modify_hdr; 312 313 restore->rule = esw_add_restore_rule(esw, obj_id); 314 if (IS_ERR(restore->rule)) { 315 err = PTR_ERR(restore->rule); 316 goto err_restore; 317 } 318 319 hash_add(tc_psample->restore_hashtbl, &restore->hlist, hash_key); 320 add_ref: 321 restore->count++; 322 mutex_unlock(&tc_psample->restore_lock); 323 return restore; 324 325 err_restore: 326 mlx5_modify_header_dealloc(mdev, restore->modify_hdr); 327 err_modify_hdr: 328 kfree(restore); 329 err_alloc: 330 mutex_unlock(&tc_psample->restore_lock); 331 return ERR_PTR(err); 332 } 333 334 static void 335 sample_restore_put(struct mlx5e_tc_psample *tc_psample, struct mlx5e_sample_restore *restore) 336 { 337 mutex_lock(&tc_psample->restore_lock); 338 if (--restore->count == 0) 339 hash_del(&restore->hlist); 340 mutex_unlock(&tc_psample->restore_lock); 341 342 if (!restore->count) { 343 mlx5_del_flow_rules(restore->rule); 344 mlx5_modify_header_dealloc(tc_psample->esw->dev, restore->modify_hdr); 345 kfree(restore); 346 } 347 } 348 349 void mlx5e_tc_sample_skb(struct sk_buff *skb, struct mlx5_mapped_obj *mapped_obj) 350 { 351 u32 trunc_size = mapped_obj->sample.trunc_size; 352 struct psample_group psample_group = {}; 353 struct psample_metadata md = {}; 354 355 md.trunc_size = trunc_size ? min(trunc_size, skb->len) : skb->len; 356 md.in_ifindex = skb->dev->ifindex; 357 psample_group.group_num = mapped_obj->sample.group_id; 358 psample_group.net = &init_net; 359 skb_push(skb, skb->mac_len); 360 361 psample_sample_packet(&psample_group, skb, mapped_obj->sample.rate, &md); 362 } 363 364 static int 365 add_post_rule(struct mlx5_eswitch *esw, struct mlx5e_sample_flow *sample_flow, 366 struct mlx5_flow_spec *spec, struct mlx5_flow_attr *attr, 367 u32 *default_tbl_id) 368 { 369 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 370 u32 attr_sz = ns_to_attr_sz(MLX5_FLOW_NAMESPACE_FDB); 371 struct mlx5_vport_tbl_attr per_vport_tbl_attr; 372 struct mlx5_flow_table *default_tbl; 373 struct mlx5_flow_attr *post_attr; 374 int err; 375 376 /* Allocate default table per vport, chain and prio. Otherwise, there is 377 * only one default table for the same sampler object. Rules with different 378 * prio and chain may overlap. For CT sample action, per vport default 379 * table is needed to resotre the metadata. 380 */ 381 per_vport_tbl_attr.chain = attr->chain; 382 per_vport_tbl_attr.prio = attr->prio; 383 per_vport_tbl_attr.vport = esw_attr->in_rep->vport; 384 per_vport_tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns; 385 default_tbl = mlx5_esw_vporttbl_get(esw, &per_vport_tbl_attr); 386 if (IS_ERR(default_tbl)) { 387 err = PTR_ERR(default_tbl); 388 goto err_default_tbl; 389 } 390 *default_tbl_id = default_tbl->id; 391 392 post_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB); 393 if (!post_attr) { 394 err = -ENOMEM; 395 goto err_attr; 396 } 397 sample_flow->post_attr = post_attr; 398 memcpy(post_attr, attr, attr_sz); 399 /* Perform the original matches on the default table. 400 * Offload all actions except the sample action. 401 */ 402 post_attr->chain = 0; 403 post_attr->prio = 0; 404 post_attr->ft = default_tbl; 405 post_attr->flags = MLX5_ESW_ATTR_FLAG_NO_IN_PORT; 406 407 /* When offloading sample and encap action, if there is no valid 408 * neigh data struct, a slow path rule is offloaded first. Source 409 * port metadata match is set at that time. A per vport table is 410 * already allocated. No need to match it again. So clear the source 411 * port metadata match. 412 */ 413 mlx5_eswitch_clear_rule_source_port(esw, spec); 414 sample_flow->post_rule = mlx5_eswitch_add_offloaded_rule(esw, spec, post_attr); 415 if (IS_ERR(sample_flow->post_rule)) { 416 err = PTR_ERR(sample_flow->post_rule); 417 goto err_rule; 418 } 419 return 0; 420 421 err_rule: 422 kfree(post_attr); 423 err_attr: 424 mlx5_esw_vporttbl_put(esw, &per_vport_tbl_attr); 425 err_default_tbl: 426 return err; 427 } 428 429 static void 430 del_post_rule(struct mlx5_eswitch *esw, struct mlx5e_sample_flow *sample_flow, 431 struct mlx5_flow_attr *attr) 432 { 433 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 434 struct mlx5_vport_tbl_attr tbl_attr; 435 436 mlx5_eswitch_del_offloaded_rule(esw, sample_flow->post_rule, sample_flow->post_attr); 437 kfree(sample_flow->post_attr); 438 tbl_attr.chain = attr->chain; 439 tbl_attr.prio = attr->prio; 440 tbl_attr.vport = esw_attr->in_rep->vport; 441 tbl_attr.vport_ns = &mlx5_esw_vport_tbl_sample_ns; 442 mlx5_esw_vporttbl_put(esw, &tbl_attr); 443 } 444 445 /* For the following typical flow table: 446 * 447 * +-------------------------------+ 448 * + original flow table + 449 * +-------------------------------+ 450 * + original match + 451 * +-------------------------------+ 452 * + sample action + other actions + 453 * +-------------------------------+ 454 * 455 * We translate the tc filter with sample action to the following HW model: 456 * 457 * +---------------------+ 458 * + original flow table + 459 * +---------------------+ 460 * + original match + 461 * +---------------------+ 462 * | set fte_id (if reg_c preserve cap) 463 * | do decap (if required) 464 * v 465 * +------------------------------------------------+ 466 * + Flow Sampler Object + 467 * +------------------------------------------------+ 468 * + sample ratio + 469 * +------------------------------------------------+ 470 * + sample table id | default table id + 471 * +------------------------------------------------+ 472 * | | 473 * v v 474 * +-----------------------------+ +-------------------+ 475 * + sample table + + default table + 476 * +-----------------------------+ +-------------------+ 477 * + forward to management vport + | 478 * +-----------------------------+ | 479 * +-------+------+ 480 * | |reg_c preserve cap 481 * | |or decap action 482 * v v 483 * +-----------------+ +-------------+ 484 * + per vport table + + post action + 485 * +-----------------+ +-------------+ 486 * + original match + 487 * +-----------------+ 488 * + other actions + 489 * +-----------------+ 490 */ 491 struct mlx5_flow_handle * 492 mlx5e_tc_sample_offload(struct mlx5e_tc_psample *tc_psample, 493 struct mlx5_flow_spec *spec, 494 struct mlx5_flow_attr *attr, 495 u32 tunnel_id) 496 { 497 struct mlx5e_post_act_handle *post_act_handle = NULL; 498 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 499 struct mlx5_esw_flow_attr *pre_esw_attr; 500 struct mlx5_mapped_obj restore_obj = {}; 501 struct mlx5e_sample_flow *sample_flow; 502 struct mlx5e_sample_attr *sample_attr; 503 struct mlx5_flow_attr *pre_attr; 504 struct mlx5_eswitch *esw; 505 u32 default_tbl_id; 506 u32 obj_id; 507 int err; 508 509 if (IS_ERR_OR_NULL(tc_psample)) 510 return ERR_PTR(-EOPNOTSUPP); 511 512 sample_flow = kzalloc(sizeof(*sample_flow), GFP_KERNEL); 513 if (!sample_flow) 514 return ERR_PTR(-ENOMEM); 515 sample_attr = attr->sample_attr; 516 sample_attr->sample_flow = sample_flow; 517 518 /* For NICs with reg_c_preserve support or decap action, use 519 * post action instead of the per vport, chain and prio table. 520 * Only match the fte id instead of the same match in the 521 * original flow table. 522 */ 523 esw = tc_psample->esw; 524 if (MLX5_CAP_GEN(esw->dev, reg_c_preserve) || 525 attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) { 526 struct mlx5_flow_table *ft; 527 528 ft = mlx5e_tc_post_act_get_ft(tc_psample->post_act); 529 default_tbl_id = ft->id; 530 post_act_handle = mlx5e_tc_post_act_add(tc_psample->post_act, attr); 531 if (IS_ERR(post_act_handle)) { 532 err = PTR_ERR(post_act_handle); 533 goto err_post_act; 534 } 535 sample_flow->post_act_handle = post_act_handle; 536 } else { 537 err = add_post_rule(esw, sample_flow, spec, attr, &default_tbl_id); 538 if (err) 539 goto err_post_rule; 540 } 541 542 /* Create sampler object. */ 543 sample_flow->sampler = sampler_get(tc_psample, sample_attr->rate, default_tbl_id); 544 if (IS_ERR(sample_flow->sampler)) { 545 err = PTR_ERR(sample_flow->sampler); 546 goto err_sampler; 547 } 548 549 /* Create an id mapping reg_c0 value to sample object. */ 550 restore_obj.type = MLX5_MAPPED_OBJ_SAMPLE; 551 restore_obj.sample.group_id = sample_attr->group_num; 552 restore_obj.sample.rate = sample_attr->rate; 553 restore_obj.sample.trunc_size = sample_attr->trunc_size; 554 restore_obj.sample.tunnel_id = tunnel_id; 555 err = mapping_add(esw->offloads.reg_c0_obj_pool, &restore_obj, &obj_id); 556 if (err) 557 goto err_obj_id; 558 sample_attr->restore_obj_id = obj_id; 559 560 /* Create sample restore context. */ 561 sample_flow->restore = sample_restore_get(tc_psample, obj_id, post_act_handle); 562 if (IS_ERR(sample_flow->restore)) { 563 err = PTR_ERR(sample_flow->restore); 564 goto err_sample_restore; 565 } 566 567 /* Perform the original matches on the original table. Offload the 568 * sample action. The destination is the sampler object. 569 */ 570 pre_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB); 571 if (!pre_attr) { 572 err = -ENOMEM; 573 goto err_alloc_pre_flow_attr; 574 } 575 pre_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 576 /* For decap action, do decap in the original flow table instead of the 577 * default flow table. 578 */ 579 if (tunnel_id) 580 pre_attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 581 pre_attr->modify_hdr = sample_flow->restore->modify_hdr; 582 pre_attr->flags = MLX5_ESW_ATTR_FLAG_SAMPLE; 583 pre_attr->inner_match_level = attr->inner_match_level; 584 pre_attr->outer_match_level = attr->outer_match_level; 585 pre_attr->chain = attr->chain; 586 pre_attr->prio = attr->prio; 587 pre_attr->sample_attr = attr->sample_attr; 588 sample_attr->sampler_id = sample_flow->sampler->sampler_id; 589 pre_esw_attr = pre_attr->esw_attr; 590 pre_esw_attr->in_mdev = esw_attr->in_mdev; 591 pre_esw_attr->in_rep = esw_attr->in_rep; 592 sample_flow->pre_rule = mlx5_eswitch_add_offloaded_rule(esw, spec, pre_attr); 593 if (IS_ERR(sample_flow->pre_rule)) { 594 err = PTR_ERR(sample_flow->pre_rule); 595 goto err_pre_offload_rule; 596 } 597 sample_flow->pre_attr = pre_attr; 598 599 return sample_flow->pre_rule; 600 601 err_pre_offload_rule: 602 kfree(pre_attr); 603 err_alloc_pre_flow_attr: 604 sample_restore_put(tc_psample, sample_flow->restore); 605 err_sample_restore: 606 mapping_remove(esw->offloads.reg_c0_obj_pool, obj_id); 607 err_obj_id: 608 sampler_put(tc_psample, sample_flow->sampler); 609 err_sampler: 610 if (sample_flow->post_rule) 611 del_post_rule(esw, sample_flow, attr); 612 err_post_rule: 613 if (post_act_handle) 614 mlx5e_tc_post_act_del(tc_psample->post_act, post_act_handle); 615 err_post_act: 616 kfree(sample_flow); 617 return ERR_PTR(err); 618 } 619 620 void 621 mlx5e_tc_sample_unoffload(struct mlx5e_tc_psample *tc_psample, 622 struct mlx5_flow_handle *rule, 623 struct mlx5_flow_attr *attr) 624 { 625 struct mlx5e_sample_flow *sample_flow; 626 struct mlx5_eswitch *esw; 627 628 if (IS_ERR_OR_NULL(tc_psample)) 629 return; 630 631 /* The following delete order can't be changed, otherwise, 632 * will hit fw syndromes. 633 */ 634 esw = tc_psample->esw; 635 sample_flow = attr->sample_attr->sample_flow; 636 mlx5_eswitch_del_offloaded_rule(esw, sample_flow->pre_rule, sample_flow->pre_attr); 637 638 sample_restore_put(tc_psample, sample_flow->restore); 639 mapping_remove(esw->offloads.reg_c0_obj_pool, attr->sample_attr->restore_obj_id); 640 sampler_put(tc_psample, sample_flow->sampler); 641 if (sample_flow->post_act_handle) 642 mlx5e_tc_post_act_del(tc_psample->post_act, sample_flow->post_act_handle); 643 else 644 del_post_rule(esw, sample_flow, attr); 645 646 kfree(sample_flow->pre_attr); 647 kfree(sample_flow); 648 } 649 650 struct mlx5e_tc_psample * 651 mlx5e_tc_sample_init(struct mlx5_eswitch *esw, struct mlx5e_post_act *post_act) 652 { 653 struct mlx5e_tc_psample *tc_psample; 654 int err; 655 656 tc_psample = kzalloc(sizeof(*tc_psample), GFP_KERNEL); 657 if (!tc_psample) 658 return ERR_PTR(-ENOMEM); 659 if (IS_ERR_OR_NULL(post_act)) { 660 err = PTR_ERR(post_act); 661 goto err_post_act; 662 } 663 tc_psample->post_act = post_act; 664 tc_psample->esw = esw; 665 err = sampler_termtbl_create(tc_psample); 666 if (err) 667 goto err_post_act; 668 669 mutex_init(&tc_psample->ht_lock); 670 mutex_init(&tc_psample->restore_lock); 671 672 return tc_psample; 673 674 err_post_act: 675 kfree(tc_psample); 676 return ERR_PTR(err); 677 } 678 679 void 680 mlx5e_tc_sample_cleanup(struct mlx5e_tc_psample *tc_psample) 681 { 682 if (IS_ERR_OR_NULL(tc_psample)) 683 return; 684 685 mutex_destroy(&tc_psample->restore_lock); 686 mutex_destroy(&tc_psample->ht_lock); 687 sampler_termtbl_destroy(tc_psample); 688 kfree(tc_psample); 689 } 690