538 		return root->dev->priv.steering;  in get_steering()
690 	struct mlx5_flow_steering *steering = get_steering(node);  in del_sw_fte()  local
703 	kmem_cache_free(steering->ftes_cache, fte);  in del_sw_fte()
726 	struct mlx5_flow_steering *steering = get_steering(node);  in del_sw_flow_group()  local
744 	kmem_cache_free(steering->fgs_cache, fg);  in del_sw_flow_group()
776 	struct mlx5_flow_steering *steering = get_steering(&ft->node);  in alloc_fte()  local
779 	fte = kmem_cache_zalloc(steering->ftes_cache, GFP_KERNEL);  in alloc_fte()
793 static void dealloc_flow_group(struct mlx5_flow_steering *steering,  in dealloc_flow_group()  argument
797 	kmem_cache_free(steering->fgs_cache, fg);  in dealloc_flow_group()
800 static struct mlx5_flow_group *alloc_flow_group(struct mlx5_flow_steering *steering,  in alloc_flow_group()  argument
809 	fg = kmem_cache_zalloc(steering->fgs_cache, GFP_KERNEL);  in alloc_flow_group()
815 		kmem_cache_free(steering->fgs_cache, fg);  in alloc_flow_group()
837 	struct mlx5_flow_steering *steering = get_steering(&ft->node);  in alloc_insert_flow_group()  local
841 	fg = alloc_flow_group(steering, match_criteria_enable, match_criteria,  in alloc_insert_flow_group()
851 		dealloc_flow_group(steering, fg);  in alloc_insert_flow_group()
1973 	struct mlx5_flow_steering *steering = get_steering(&ft->node);  in try_add_to_existing_fg()  local
2004 		kmem_cache_free(steering->ftes_cache, fte);  in try_add_to_existing_fg()
2046 			kmem_cache_free(steering->ftes_cache, fte);  in try_add_to_existing_fg()
2060 	kmem_cache_free(steering->ftes_cache, fte);  in try_add_to_existing_fg()
2072 	struct mlx5_flow_steering *steering = get_steering(&ft->node);  in _mlx5_add_flow_rules()  local
2167 	kmem_cache_free(steering->ftes_cache, fte);  in _mlx5_add_flow_rules()
2409 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_get_fdb_sub_ns()  local
2411 	if (!steering || !steering->fdb_sub_ns)  in mlx5_get_fdb_sub_ns()
2414 	return steering->fdb_sub_ns[n];  in mlx5_get_fdb_sub_ns()
2438 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_get_flow_namespace()  local
2444 	if (!steering)  in mlx5_get_flow_namespace()
2449 		if (steering->fdb_root_ns)  in mlx5_get_flow_namespace()
2450 			return &steering->fdb_root_ns->ns;  in mlx5_get_flow_namespace()
2453 		if (steering->port_sel_root_ns)  in mlx5_get_flow_namespace()
2454 			return &steering->port_sel_root_ns->ns;  in mlx5_get_flow_namespace()
2457 		if (steering->sniffer_rx_root_ns)  in mlx5_get_flow_namespace()
2458 			return &steering->sniffer_rx_root_ns->ns;  in mlx5_get_flow_namespace()
2461 		if (steering->sniffer_tx_root_ns)  in mlx5_get_flow_namespace()
2462 			return &steering->sniffer_tx_root_ns->ns;  in mlx5_get_flow_namespace()
2465 		root_ns = steering->fdb_root_ns;  in mlx5_get_flow_namespace()
2471 		root_ns = steering->egress_root_ns;  in mlx5_get_flow_namespace()
2475 		root_ns = steering->rdma_rx_root_ns;  in mlx5_get_flow_namespace()
2479 		root_ns = steering->rdma_rx_root_ns;  in mlx5_get_flow_namespace()
2483 		root_ns = steering->rdma_tx_root_ns;  in mlx5_get_flow_namespace()
2486 		root_ns = steering->rdma_rx_root_ns;  in mlx5_get_flow_namespace()
2490 		root_ns = steering->rdma_tx_root_ns;  in mlx5_get_flow_namespace()
2494 		root_ns = steering->rdma_rx_root_ns;  in mlx5_get_flow_namespace()
2498 		root_ns = steering->rdma_tx_root_ns;  in mlx5_get_flow_namespace()
2502 		root_ns = steering->rdma_rx_root_ns;  in mlx5_get_flow_namespace()
2506 		root_ns = steering->rdma_tx_root_ns;  in mlx5_get_flow_namespace()
2511 		root_ns = steering->root_ns;  in mlx5_get_flow_namespace()
2535 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_get_flow_vport_acl_namespace()  local
2537 	if (!steering)  in mlx5_get_flow_vport_acl_namespace()
2542 		if (vport >= steering->esw_egress_acl_vports)  in mlx5_get_flow_vport_acl_namespace()
2544 		if (steering->esw_egress_root_ns &&  in mlx5_get_flow_vport_acl_namespace()
2545 		    steering->esw_egress_root_ns[vport])  in mlx5_get_flow_vport_acl_namespace()
2546 			return &steering->esw_egress_root_ns[vport]->ns;  in mlx5_get_flow_vport_acl_namespace()
2550 		if (vport >= steering->esw_ingress_acl_vports)  in mlx5_get_flow_vport_acl_namespace()
2552 		if (steering->esw_ingress_root_ns &&  in mlx5_get_flow_vport_acl_namespace()
2553 		    steering->esw_ingress_root_ns[vport])  in mlx5_get_flow_vport_acl_namespace()
2554 			return &steering->esw_ingress_root_ns[vport]->ns;  in mlx5_get_flow_vport_acl_namespace()
2652 static int init_root_tree_recursive(struct mlx5_flow_steering *steering,  in init_root_tree_recursive()  argument
2658 	int max_ft_level = MLX5_CAP_FLOWTABLE(steering->dev,  in init_root_tree_recursive()
2669 		    !has_required_caps(steering->dev, &init_node->caps))  in init_root_tree_recursive()
2690 		err = init_root_tree_recursive(steering, &init_node->children[i],  in init_root_tree_recursive()
2703 static int init_root_tree(struct mlx5_flow_steering *steering,  in init_root_tree()  argument
2711 		err = init_root_tree_recursive(steering, &init_node->children[i],  in init_root_tree()
2732 *create_root_ns(struct mlx5_flow_steering *steering,  in create_root_ns()  argument
2744 	root_ns->dev = steering->dev;  in create_root_ns()
2810 static int create_anchor_flow_table(struct mlx5_flow_steering *steering)  in create_anchor_flow_table()  argument
2816 	ns = mlx5_get_flow_namespace(steering->dev, MLX5_FLOW_NAMESPACE_ANCHOR);  in create_anchor_flow_table()
2826 		mlx5_core_err(steering->dev, "Failed to create last anchor flow table");  in create_anchor_flow_table()
2832 static int init_root_ns(struct mlx5_flow_steering *steering)  in init_root_ns()  argument
2836 	steering->root_ns = create_root_ns(steering, FS_FT_NIC_RX);  in init_root_ns()
2837 	if (!steering->root_ns)  in init_root_ns()
2840 	err = init_root_tree(steering, &root_fs, &steering->root_ns->ns.node);  in init_root_ns()
2844 	set_prio_attrs(steering->root_ns);  in init_root_ns()
2845 	err = create_anchor_flow_table(steering);  in init_root_ns()
2852 	cleanup_root_ns(steering->root_ns);  in init_root_ns()
2853 	steering->root_ns = NULL;  in init_root_ns()
2879 static int init_sniffer_tx_root_ns(struct mlx5_flow_steering *steering)  in init_sniffer_tx_root_ns()  argument
2883 	steering->sniffer_tx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_TX);  in init_sniffer_tx_root_ns()
2884 	if (!steering->sniffer_tx_root_ns)  in init_sniffer_tx_root_ns()
2888 	prio = fs_create_prio(&steering->sniffer_tx_root_ns->ns, 0, 1);  in init_sniffer_tx_root_ns()
2892 static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering)  in init_sniffer_rx_root_ns()  argument
2896 	steering->sniffer_rx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_RX);  in init_sniffer_rx_root_ns()
2897 	if (!steering->sniffer_rx_root_ns)  in init_sniffer_rx_root_ns()
2901 	prio = fs_create_prio(&steering->sniffer_rx_root_ns->ns, 0, 1);  in init_sniffer_rx_root_ns()
2906 static int init_port_sel_root_ns(struct mlx5_flow_steering *steering)  in init_port_sel_root_ns()  argument
2910 	steering->port_sel_root_ns = create_root_ns(steering, FS_FT_PORT_SEL);  in init_port_sel_root_ns()
2911 	if (!steering->port_sel_root_ns)  in init_port_sel_root_ns()
2915 	prio = fs_create_prio(&steering->port_sel_root_ns->ns, 0,  in init_port_sel_root_ns()
2920 static int init_rdma_rx_root_ns(struct mlx5_flow_steering *steering)  in init_rdma_rx_root_ns()  argument
2924 	steering->rdma_rx_root_ns = create_root_ns(steering, FS_FT_RDMA_RX);  in init_rdma_rx_root_ns()
2925 	if (!steering->rdma_rx_root_ns)  in init_rdma_rx_root_ns()
2928 	err = init_root_tree(steering, &rdma_rx_root_fs,  in init_rdma_rx_root_ns()
2929 			     &steering->rdma_rx_root_ns->ns.node);  in init_rdma_rx_root_ns()
2933 	set_prio_attrs(steering->rdma_rx_root_ns);  in init_rdma_rx_root_ns()
2938 	cleanup_root_ns(steering->rdma_rx_root_ns);  in init_rdma_rx_root_ns()
2939 	steering->rdma_rx_root_ns = NULL;  in init_rdma_rx_root_ns()
2943 static int init_rdma_tx_root_ns(struct mlx5_flow_steering *steering)  in init_rdma_tx_root_ns()  argument
2947 	steering->rdma_tx_root_ns = create_root_ns(steering, FS_FT_RDMA_TX);  in init_rdma_tx_root_ns()
2948 	if (!steering->rdma_tx_root_ns)  in init_rdma_tx_root_ns()
2951 	err = init_root_tree(steering, &rdma_tx_root_fs,  in init_rdma_tx_root_ns()
2952 			     &steering->rdma_tx_root_ns->ns.node);  in init_rdma_tx_root_ns()
2956 	set_prio_attrs(steering->rdma_tx_root_ns);  in init_rdma_tx_root_ns()
2961 	cleanup_root_ns(steering->rdma_tx_root_ns);  in init_rdma_tx_root_ns()
2962 	steering->rdma_tx_root_ns = NULL;  in init_rdma_tx_root_ns()
2971 static void store_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,  in store_fdb_sub_ns_prio_chain()  argument
2976 	while (steering->fdb_sub_ns[chain])  in store_fdb_sub_ns_prio_chain()
2979 	steering->fdb_sub_ns[chain] = ns;  in store_fdb_sub_ns_prio_chain()
2982 static int create_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,  in create_fdb_sub_ns_prio_chain()  argument
2999 	store_fdb_sub_ns_prio_chain(steering, ns);  in create_fdb_sub_ns_prio_chain()
3004 static int create_fdb_chains(struct mlx5_flow_steering *steering,  in create_fdb_chains()  argument
3014 	maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns,  in create_fdb_chains()
3021 		err = create_fdb_sub_ns_prio_chain(steering, maj_prio);  in create_fdb_chains()
3029 static int create_fdb_fast_path(struct mlx5_flow_steering *steering)  in create_fdb_fast_path()  argument
3033 	steering->fdb_sub_ns = kcalloc(FDB_NUM_CHAINS,  in create_fdb_fast_path()
3034 				       sizeof(*steering->fdb_sub_ns),  in create_fdb_fast_path()
3036 	if (!steering->fdb_sub_ns)  in create_fdb_fast_path()
3039 	err = create_fdb_chains(steering, FDB_TC_OFFLOAD, FDB_TC_MAX_CHAIN + 1);  in create_fdb_fast_path()
3043 	err = create_fdb_chains(steering, FDB_FT_OFFLOAD, 1);  in create_fdb_fast_path()
3050 static int create_fdb_bypass(struct mlx5_flow_steering *steering)  in create_fdb_bypass()  argument
3056 	prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BYPASS_PATH, 0);  in create_fdb_bypass()
3072 static void cleanup_fdb_root_ns(struct mlx5_flow_steering *steering)  in cleanup_fdb_root_ns()  argument
3074 	cleanup_root_ns(steering->fdb_root_ns);  in cleanup_fdb_root_ns()
3075 	steering->fdb_root_ns = NULL;  in cleanup_fdb_root_ns()
3076 	kfree(steering->fdb_sub_ns);  in cleanup_fdb_root_ns()
3077 	steering->fdb_sub_ns = NULL;  in cleanup_fdb_root_ns()
3080 static int init_fdb_root_ns(struct mlx5_flow_steering *steering)  in init_fdb_root_ns()  argument
3085 	steering->fdb_root_ns = create_root_ns(steering, FS_FT_FDB);  in init_fdb_root_ns()
3086 	if (!steering->fdb_root_ns)  in init_fdb_root_ns()
3089 	err = create_fdb_bypass(steering);  in init_fdb_root_ns()
3093 	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_CRYPTO_INGRESS, 3);  in init_fdb_root_ns()
3099 	err = create_fdb_fast_path(steering);  in init_fdb_root_ns()
3103 	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_TC_MISS, 1);  in init_fdb_root_ns()
3109 	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BR_OFFLOAD, 4);  in init_fdb_root_ns()
3115 	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1);  in init_fdb_root_ns()
3121 	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_CRYPTO_EGRESS, 3);  in init_fdb_root_ns()
3132 	maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_PER_VPORT, 1);  in init_fdb_root_ns()
3138 	set_prio_attrs(steering->fdb_root_ns);  in init_fdb_root_ns()
3142 	cleanup_fdb_root_ns(steering);  in init_fdb_root_ns()
3146 static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)  in init_egress_acl_root_ns()  argument
3150 	steering->esw_egress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL);  in init_egress_acl_root_ns()
3151 	if (!steering->esw_egress_root_ns[vport])  in init_egress_acl_root_ns()
3155 	prio = fs_create_prio(&steering->esw_egress_root_ns[vport]->ns, 0, 1);  in init_egress_acl_root_ns()
3159 static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)  in init_ingress_acl_root_ns()  argument
3163 	steering->esw_ingress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL);  in init_ingress_acl_root_ns()
3164 	if (!steering->esw_ingress_root_ns[vport])  in init_ingress_acl_root_ns()
3168 	prio = fs_create_prio(&steering->esw_ingress_root_ns[vport]->ns, 0, 1);  in init_ingress_acl_root_ns()
3174 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_fs_egress_acls_init()  local
3178 	steering->esw_egress_root_ns =  in mlx5_fs_egress_acls_init()
3180 				sizeof(*steering->esw_egress_root_ns),  in mlx5_fs_egress_acls_init()
3182 	if (!steering->esw_egress_root_ns)  in mlx5_fs_egress_acls_init()
3186 		err = init_egress_acl_root_ns(steering, i);  in mlx5_fs_egress_acls_init()
3190 	steering->esw_egress_acl_vports = total_vports;  in mlx5_fs_egress_acls_init()
3195 		cleanup_root_ns(steering->esw_egress_root_ns[i]);  in mlx5_fs_egress_acls_init()
3196 	kfree(steering->esw_egress_root_ns);  in mlx5_fs_egress_acls_init()
3197 	steering->esw_egress_root_ns = NULL;  in mlx5_fs_egress_acls_init()
3203 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_fs_egress_acls_cleanup()  local
3206 	if (!steering->esw_egress_root_ns)  in mlx5_fs_egress_acls_cleanup()
3209 	for (i = 0; i < steering->esw_egress_acl_vports; i++)  in mlx5_fs_egress_acls_cleanup()
3210 		cleanup_root_ns(steering->esw_egress_root_ns[i]);  in mlx5_fs_egress_acls_cleanup()
3212 	kfree(steering->esw_egress_root_ns);  in mlx5_fs_egress_acls_cleanup()
3213 	steering->esw_egress_root_ns = NULL;  in mlx5_fs_egress_acls_cleanup()
3218 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_fs_ingress_acls_init()  local
3222 	steering->esw_ingress_root_ns =  in mlx5_fs_ingress_acls_init()
3224 				sizeof(*steering->esw_ingress_root_ns),  in mlx5_fs_ingress_acls_init()
3226 	if (!steering->esw_ingress_root_ns)  in mlx5_fs_ingress_acls_init()
3230 		err = init_ingress_acl_root_ns(steering, i);  in mlx5_fs_ingress_acls_init()
3234 	steering->esw_ingress_acl_vports = total_vports;  in mlx5_fs_ingress_acls_init()
3239 		cleanup_root_ns(steering->esw_ingress_root_ns[i]);  in mlx5_fs_ingress_acls_init()
3240 	kfree(steering->esw_ingress_root_ns);  in mlx5_fs_ingress_acls_init()
3241 	steering->esw_ingress_root_ns = NULL;  in mlx5_fs_ingress_acls_init()
3247 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_fs_ingress_acls_cleanup()  local
3250 	if (!steering->esw_ingress_root_ns)  in mlx5_fs_ingress_acls_cleanup()
3253 	for (i = 0; i < steering->esw_ingress_acl_vports; i++)  in mlx5_fs_ingress_acls_cleanup()
3254 		cleanup_root_ns(steering->esw_ingress_root_ns[i]);  in mlx5_fs_ingress_acls_cleanup()
3256 	kfree(steering->esw_ingress_root_ns);  in mlx5_fs_ingress_acls_cleanup()
3257 	steering->esw_ingress_root_ns = NULL;  in mlx5_fs_ingress_acls_cleanup()
3276 static int init_egress_root_ns(struct mlx5_flow_steering *steering)  in init_egress_root_ns()  argument
3280 	steering->egress_root_ns = create_root_ns(steering,  in init_egress_root_ns()
3282 	if (!steering->egress_root_ns)  in init_egress_root_ns()
3285 	err = init_root_tree(steering, &egress_root_fs,  in init_egress_root_ns()
3286 			     &steering->egress_root_ns->ns.node);  in init_egress_root_ns()
3289 	set_prio_attrs(steering->egress_root_ns);  in init_egress_root_ns()
3292 	cleanup_root_ns(steering->egress_root_ns);  in init_egress_root_ns()
3293 	steering->egress_root_ns = NULL;  in init_egress_root_ns()
3344 	dev->priv.steering->mode = mode;  in mlx5_fs_mode_set()
3354 	if (dev->priv.steering->mode == MLX5_FLOW_STEERING_MODE_SMFS)  in mlx5_fs_mode_get()
3371 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_fs_core_cleanup()  local
3373 	cleanup_root_ns(steering->root_ns);  in mlx5_fs_core_cleanup()
3374 	cleanup_fdb_root_ns(steering);  in mlx5_fs_core_cleanup()
3375 	cleanup_root_ns(steering->port_sel_root_ns);  in mlx5_fs_core_cleanup()
3376 	cleanup_root_ns(steering->sniffer_rx_root_ns);  in mlx5_fs_core_cleanup()
3377 	cleanup_root_ns(steering->sniffer_tx_root_ns);  in mlx5_fs_core_cleanup()
3378 	cleanup_root_ns(steering->rdma_rx_root_ns);  in mlx5_fs_core_cleanup()
3379 	cleanup_root_ns(steering->rdma_tx_root_ns);  in mlx5_fs_core_cleanup()
3380 	cleanup_root_ns(steering->egress_root_ns);  in mlx5_fs_core_cleanup()
3388 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_fs_core_init()  local
3401 		err = init_root_ns(steering);  in mlx5_fs_core_init()
3408 			err = init_fdb_root_ns(steering);  in mlx5_fs_core_init()
3415 		err = init_sniffer_rx_root_ns(steering);  in mlx5_fs_core_init()
3421 		err = init_sniffer_tx_root_ns(steering);  in mlx5_fs_core_init()
3427 		err = init_port_sel_root_ns(steering);  in mlx5_fs_core_init()
3434 		err = init_rdma_rx_root_ns(steering);  in mlx5_fs_core_init()
3440 		err = init_rdma_tx_root_ns(steering);  in mlx5_fs_core_init()
3446 		err = init_egress_root_ns(steering);  in mlx5_fs_core_init()
3460 	struct mlx5_flow_steering *steering = dev->priv.steering;  in mlx5_fs_core_free()  local
3462 	kmem_cache_destroy(steering->ftes_cache);  in mlx5_fs_core_free()
3463 	kmem_cache_destroy(steering->fgs_cache);  in mlx5_fs_core_free()
3464 	kfree(steering);  in mlx5_fs_core_free()
3471 	struct mlx5_flow_steering *steering;  in mlx5_fs_core_alloc()  local
3482 	steering = kzalloc(sizeof(*steering), GFP_KERNEL);  in mlx5_fs_core_alloc()
3483 	if (!steering) {  in mlx5_fs_core_alloc()
3488 	steering->dev = dev;  in mlx5_fs_core_alloc()
3489 	dev->priv.steering = steering;  in mlx5_fs_core_alloc()
3492 		steering->mode = MLX5_FLOW_STEERING_MODE_SMFS;  in mlx5_fs_core_alloc()
3494 		steering->mode = MLX5_FLOW_STEERING_MODE_DMFS;  in mlx5_fs_core_alloc()
3496 	steering->fgs_cache = kmem_cache_create("mlx5_fs_fgs",  in mlx5_fs_core_alloc()
3499 	steering->ftes_cache = kmem_cache_create("mlx5_fs_ftes", sizeof(struct fs_fte), 0,  in mlx5_fs_core_alloc()
3501 	if (!steering->ftes_cache || !steering->fgs_cache) {  in mlx5_fs_core_alloc()
3515 	struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;  in mlx5_fs_add_rx_underlay_qpn()
3554 	struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;  in mlx5_fs_remove_rx_underlay_qpn()