1 /*
2  * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #ifndef _MLX5_FS_CORE_
34 #define _MLX5_FS_CORE_
35 
36 #include <linux/refcount.h>
37 #include <linux/mlx5/fs.h>
38 #include <linux/rhashtable.h>
39 #include <linux/llist.h>
40 #include <steering/fs_dr.h>
41 
42 #define FDB_TC_MAX_CHAIN 3
43 #define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1)
44 #define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1)
45 
46 /* The index of the last real chain (FT) + 1 as chain zero is valid as well */
47 #define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1)
48 
49 #define FDB_TC_MAX_PRIO 16
50 #define FDB_TC_LEVELS_PER_PRIO 2
51 
52 struct mlx5_flow_definer {
53 	enum mlx5_flow_namespace_type ns_type;
54 	u32 id;
55 };
56 
57 enum mlx5_flow_resource_owner {
58 	MLX5_FLOW_RESOURCE_OWNER_FW,
59 	MLX5_FLOW_RESOURCE_OWNER_SW,
60 };
61 
62 struct mlx5_modify_hdr {
63 	enum mlx5_flow_namespace_type ns_type;
64 	enum mlx5_flow_resource_owner owner;
65 	union {
66 		struct mlx5_fs_dr_action action;
67 		u32 id;
68 	};
69 };
70 
71 struct mlx5_pkt_reformat {
72 	enum mlx5_flow_namespace_type ns_type;
73 	int reformat_type; /* from mlx5_ifc */
74 	enum mlx5_flow_resource_owner owner;
75 	union {
76 		struct mlx5_fs_dr_action action;
77 		u32 id;
78 	};
79 };
80 
81 /* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only,
82  * and those are in parallel to one another when going over them to connect
83  * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one
84  * parallel namespace will not automatically connect to the first flow table
85  * found in any prio in any next namespace, but skip the entire containing
86  * TYPE_PRIO_CHAINS prio.
87  *
88  * This is used to implement tc chains, each chain of prios is a different
89  * namespace inside a containing TYPE_PRIO_CHAINS prio.
90  */
91 
92 enum fs_node_type {
93 	FS_TYPE_NAMESPACE,
94 	FS_TYPE_PRIO,
95 	FS_TYPE_PRIO_CHAINS,
96 	FS_TYPE_FLOW_TABLE,
97 	FS_TYPE_FLOW_GROUP,
98 	FS_TYPE_FLOW_ENTRY,
99 	FS_TYPE_FLOW_DEST
100 };
101 
102 enum fs_flow_table_type {
103 	FS_FT_NIC_RX          = 0x0,
104 	FS_FT_NIC_TX          = 0x1,
105 	FS_FT_ESW_EGRESS_ACL  = 0x2,
106 	FS_FT_ESW_INGRESS_ACL = 0x3,
107 	FS_FT_FDB             = 0X4,
108 	FS_FT_SNIFFER_RX	= 0X5,
109 	FS_FT_SNIFFER_TX	= 0X6,
110 	FS_FT_RDMA_RX		= 0X7,
111 	FS_FT_RDMA_TX		= 0X8,
112 	FS_FT_PORT_SEL		= 0X9,
113 	FS_FT_MAX_TYPE = FS_FT_PORT_SEL,
114 };
115 
116 enum fs_flow_table_op_mod {
117 	FS_FT_OP_MOD_NORMAL,
118 	FS_FT_OP_MOD_LAG_DEMUX,
119 };
120 
121 enum fs_fte_status {
122 	FS_FTE_STATUS_EXISTING = 1UL << 0,
123 };
124 
125 enum mlx5_flow_steering_mode {
126 	MLX5_FLOW_STEERING_MODE_DMFS,
127 	MLX5_FLOW_STEERING_MODE_SMFS
128 };
129 
130 enum mlx5_flow_steering_capabilty {
131 	MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0,
132 	MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1,
133 	MLX5_FLOW_STEERING_CAP_MATCH_RANGES = 1UL << 2,
134 };
135 
136 struct mlx5_flow_steering {
137 	struct mlx5_core_dev *dev;
138 	enum   mlx5_flow_steering_mode	mode;
139 	struct kmem_cache		*fgs_cache;
140 	struct kmem_cache               *ftes_cache;
141 	struct mlx5_flow_root_namespace *root_ns;
142 	struct mlx5_flow_root_namespace *fdb_root_ns;
143 	struct mlx5_flow_namespace	**fdb_sub_ns;
144 	struct mlx5_flow_root_namespace **esw_egress_root_ns;
145 	struct mlx5_flow_root_namespace **esw_ingress_root_ns;
146 	struct mlx5_flow_root_namespace	*sniffer_tx_root_ns;
147 	struct mlx5_flow_root_namespace	*sniffer_rx_root_ns;
148 	struct mlx5_flow_root_namespace	*rdma_rx_root_ns;
149 	struct mlx5_flow_root_namespace	*rdma_tx_root_ns;
150 	struct mlx5_flow_root_namespace	*egress_root_ns;
151 	struct mlx5_flow_root_namespace	*port_sel_root_ns;
152 	int esw_egress_acl_vports;
153 	int esw_ingress_acl_vports;
154 };
155 
156 struct fs_node {
157 	struct list_head	list;
158 	struct list_head	children;
159 	enum fs_node_type	type;
160 	struct fs_node		*parent;
161 	struct fs_node		*root;
162 	/* lock the node for writing and traversing */
163 	struct rw_semaphore	lock;
164 	refcount_t		refcount;
165 	bool			active;
166 	void			(*del_hw_func)(struct fs_node *);
167 	void			(*del_sw_func)(struct fs_node *);
168 	atomic_t		version;
169 };
170 
171 struct mlx5_flow_rule {
172 	struct fs_node				node;
173 	struct mlx5_flow_table			*ft;
174 	struct mlx5_flow_destination		dest_attr;
175 	/* next_ft should be accessed under chain_lock and only of
176 	 * destination type is FWD_NEXT_fT.
177 	 */
178 	struct list_head			next_ft;
179 	u32					sw_action;
180 };
181 
182 struct mlx5_flow_handle {
183 	int num_rules;
184 	struct mlx5_flow_rule *rule[];
185 };
186 
187 /* Type of children is mlx5_flow_group */
188 struct mlx5_flow_table {
189 	struct fs_node			node;
190 	struct mlx5_fs_dr_table		fs_dr_table;
191 	u32				id;
192 	u16				vport;
193 	unsigned int			max_fte;
194 	unsigned int			level;
195 	enum fs_flow_table_type		type;
196 	enum fs_flow_table_op_mod	op_mod;
197 	struct {
198 		bool			active;
199 		unsigned int		required_groups;
200 		unsigned int		group_size;
201 		unsigned int		num_groups;
202 		unsigned int		max_fte;
203 	} autogroup;
204 	/* Protect fwd_rules */
205 	struct mutex			lock;
206 	/* FWD rules that point on this flow table */
207 	struct list_head		fwd_rules;
208 	u32				flags;
209 	struct rhltable			fgs_hash;
210 	enum mlx5_flow_table_miss_action def_miss_action;
211 	struct mlx5_flow_namespace	*ns;
212 };
213 
214 struct mlx5_ft_underlay_qp {
215 	struct list_head list;
216 	u32 qpn;
217 };
218 
219 #define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_e00
220 /* Calculate the fte_match_param length and without the reserved length.
221  * Make sure the reserved field is the last.
222  */
223 #define MLX5_ST_SZ_DW_MATCH_PARAM					    \
224 	((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
225 	 BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) !=		     \
226 			   MLX5_FLD_SZ_BYTES(fte_match_param,		     \
227 					     MLX5_FTE_MATCH_PARAM_RESERVED) +\
228 			   MLX5_BYTE_OFF(fte_match_param,		     \
229 					 MLX5_FTE_MATCH_PARAM_RESERVED)))
230 
231 /* Type of children is mlx5_flow_rule */
232 struct fs_fte {
233 	struct fs_node			node;
234 	struct mlx5_fs_dr_rule		fs_dr_rule;
235 	u32				val[MLX5_ST_SZ_DW_MATCH_PARAM];
236 	u32				dests_size;
237 	u32				fwd_dests;
238 	u32				index;
239 	struct mlx5_flow_context	flow_context;
240 	struct mlx5_flow_act		action;
241 	enum fs_fte_status		status;
242 	struct mlx5_fc			*counter;
243 	struct rhash_head		hash;
244 	int				modify_mask;
245 };
246 
247 /* Type of children is mlx5_flow_table/namespace */
248 struct fs_prio {
249 	struct fs_node			node;
250 	unsigned int			num_levels;
251 	unsigned int			start_level;
252 	unsigned int			prio;
253 	unsigned int			num_ft;
254 };
255 
256 /* Type of children is fs_prio */
257 struct mlx5_flow_namespace {
258 	/* parent == NULL => root ns */
259 	struct	fs_node			node;
260 	enum mlx5_flow_table_miss_action def_miss_action;
261 };
262 
263 struct mlx5_flow_group_mask {
264 	u8	match_criteria_enable;
265 	u32	match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
266 };
267 
268 /* Type of children is fs_fte */
269 struct mlx5_flow_group {
270 	struct fs_node			node;
271 	struct mlx5_fs_dr_matcher	fs_dr_matcher;
272 	struct mlx5_flow_group_mask	mask;
273 	u32				start_index;
274 	u32				max_ftes;
275 	struct ida			fte_allocator;
276 	u32				id;
277 	struct rhashtable		ftes_hash;
278 	struct rhlist_head		hash;
279 };
280 
281 struct mlx5_flow_root_namespace {
282 	struct mlx5_flow_namespace	ns;
283 	enum   mlx5_flow_steering_mode	mode;
284 	struct mlx5_fs_dr_domain	fs_dr_domain;
285 	enum   fs_flow_table_type	table_type;
286 	struct mlx5_core_dev		*dev;
287 	struct mlx5_flow_table		*root_ft;
288 	/* Should be held when chaining flow tables */
289 	struct mutex			chain_lock;
290 	struct list_head		underlay_qpns;
291 	const struct mlx5_flow_cmds	*cmds;
292 };
293 
294 int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
295 void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
296 void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
297 			      struct delayed_work *dwork,
298 			      unsigned long delay);
299 void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
300 				      unsigned long interval);
301 
302 const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
303 
304 int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
305 				 struct mlx5_flow_root_namespace *peer_ns,
306 				 u16 peer_vhca_id);
307 
308 int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
309 				 enum mlx5_flow_steering_mode mode);
310 
311 int mlx5_fs_core_alloc(struct mlx5_core_dev *dev);
312 void mlx5_fs_core_free(struct mlx5_core_dev *dev);
313 int mlx5_fs_core_init(struct mlx5_core_dev *dev);
314 void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev);
315 
316 int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports);
317 void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev);
318 int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports);
319 void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev);
320 
321 u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type);
322 
323 struct mlx5_flow_root_namespace *find_root(struct fs_node *node);
324 
325 #define fs_get_obj(v, _node)  {v = container_of((_node), typeof(*v), node); }
326 
327 #define fs_list_for_each_entry(pos, root)		\
328 	list_for_each_entry(pos, root, node.list)
329 
330 #define fs_list_for_each_entry_safe(pos, tmp, root)		\
331 	list_for_each_entry_safe(pos, tmp, root, node.list)
332 
333 #define fs_for_each_ns_or_ft_reverse(pos, prio)				\
334 	list_for_each_entry_reverse(pos, &(prio)->node.children, list)
335 
336 #define fs_for_each_ns_or_ft(pos, prio)					\
337 	list_for_each_entry(pos, (&(prio)->node.children), list)
338 
339 #define fs_for_each_prio(pos, ns)			\
340 	fs_list_for_each_entry(pos, &(ns)->node.children)
341 
342 #define fs_for_each_ns(pos, prio)			\
343 	fs_list_for_each_entry(pos, &(prio)->node.children)
344 
345 #define fs_for_each_ft(pos, prio)			\
346 	fs_list_for_each_entry(pos, &(prio)->node.children)
347 
348 #define fs_for_each_ft_safe(pos, tmp, prio)			\
349 	fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children)
350 
351 #define fs_for_each_fg(pos, ft)			\
352 	fs_list_for_each_entry(pos, &(ft)->node.children)
353 
354 #define fs_for_each_fte(pos, fg)			\
355 	fs_list_for_each_entry(pos, &(fg)->node.children)
356 
357 #define fs_for_each_dst(pos, fte)			\
358 	fs_list_for_each_entry(pos, &(fte)->node.children)
359 
360 #define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) (		\
361 	(type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) :		\
362 	(type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) :		\
363 	(type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) :		\
364 	(type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) :		\
365 	(type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) :		\
366 	(type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) :		\
367 	(type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) :		\
368 	(type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) :		\
369 	(type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) :      \
370 	(type == FS_FT_PORT_SEL) ? MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) :      \
371 	(BUILD_BUG_ON_ZERO(FS_FT_PORT_SEL != FS_FT_MAX_TYPE))\
372 	)
373 
374 #endif
375