1 /*
2  * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #ifndef _MLX5_FS_CORE_
34 #define _MLX5_FS_CORE_
35 
36 #include <linux/refcount.h>
37 #include <linux/mlx5/fs.h>
38 #include <linux/rhashtable.h>
39 #include <linux/llist.h>
40 #include <steering/fs_dr.h>
41 
42 #define FDB_TC_MAX_CHAIN 3
43 #define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1)
44 #define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1)
45 
46 /* The index of the last real chain (FT) + 1 as chain zero is valid as well */
47 #define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1)
48 
49 #define FDB_TC_MAX_PRIO 16
50 #define FDB_TC_LEVELS_PER_PRIO 2
51 
52 struct mlx5_flow_definer {
53 	enum mlx5_flow_namespace_type ns_type;
54 	u32 id;
55 };
56 
57 struct mlx5_modify_hdr {
58 	enum mlx5_flow_namespace_type ns_type;
59 	union {
60 		struct mlx5_fs_dr_action action;
61 		u32 id;
62 	};
63 };
64 
65 struct mlx5_pkt_reformat {
66 	enum mlx5_flow_namespace_type ns_type;
67 	int reformat_type; /* from mlx5_ifc */
68 	union {
69 		struct mlx5_fs_dr_action action;
70 		u32 id;
71 	};
72 };
73 
74 /* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only,
75  * and those are in parallel to one another when going over them to connect
76  * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one
77  * parallel namespace will not automatically connect to the first flow table
78  * found in any prio in any next namespace, but skip the entire containing
79  * TYPE_PRIO_CHAINS prio.
80  *
81  * This is used to implement tc chains, each chain of prios is a different
82  * namespace inside a containing TYPE_PRIO_CHAINS prio.
83  */
84 
85 enum fs_node_type {
86 	FS_TYPE_NAMESPACE,
87 	FS_TYPE_PRIO,
88 	FS_TYPE_PRIO_CHAINS,
89 	FS_TYPE_FLOW_TABLE,
90 	FS_TYPE_FLOW_GROUP,
91 	FS_TYPE_FLOW_ENTRY,
92 	FS_TYPE_FLOW_DEST
93 };
94 
95 enum fs_flow_table_type {
96 	FS_FT_NIC_RX          = 0x0,
97 	FS_FT_NIC_TX          = 0x1,
98 	FS_FT_ESW_EGRESS_ACL  = 0x2,
99 	FS_FT_ESW_INGRESS_ACL = 0x3,
100 	FS_FT_FDB             = 0X4,
101 	FS_FT_SNIFFER_RX	= 0X5,
102 	FS_FT_SNIFFER_TX	= 0X6,
103 	FS_FT_RDMA_RX		= 0X7,
104 	FS_FT_RDMA_TX		= 0X8,
105 	FS_FT_PORT_SEL		= 0X9,
106 	FS_FT_MAX_TYPE = FS_FT_PORT_SEL,
107 };
108 
109 enum fs_flow_table_op_mod {
110 	FS_FT_OP_MOD_NORMAL,
111 	FS_FT_OP_MOD_LAG_DEMUX,
112 };
113 
114 enum fs_fte_status {
115 	FS_FTE_STATUS_EXISTING = 1UL << 0,
116 };
117 
118 enum mlx5_flow_steering_mode {
119 	MLX5_FLOW_STEERING_MODE_DMFS,
120 	MLX5_FLOW_STEERING_MODE_SMFS
121 };
122 
123 enum mlx5_flow_steering_capabilty {
124 	MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0,
125 	MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1,
126 };
127 
128 struct mlx5_flow_steering {
129 	struct mlx5_core_dev *dev;
130 	enum   mlx5_flow_steering_mode	mode;
131 	struct kmem_cache		*fgs_cache;
132 	struct kmem_cache               *ftes_cache;
133 	struct mlx5_flow_root_namespace *root_ns;
134 	struct mlx5_flow_root_namespace *fdb_root_ns;
135 	struct mlx5_flow_namespace	**fdb_sub_ns;
136 	struct mlx5_flow_root_namespace **esw_egress_root_ns;
137 	struct mlx5_flow_root_namespace **esw_ingress_root_ns;
138 	struct mlx5_flow_root_namespace	*sniffer_tx_root_ns;
139 	struct mlx5_flow_root_namespace	*sniffer_rx_root_ns;
140 	struct mlx5_flow_root_namespace	*rdma_rx_root_ns;
141 	struct mlx5_flow_root_namespace	*rdma_tx_root_ns;
142 	struct mlx5_flow_root_namespace	*egress_root_ns;
143 	struct mlx5_flow_root_namespace	*port_sel_root_ns;
144 	int esw_egress_acl_vports;
145 	int esw_ingress_acl_vports;
146 };
147 
148 struct fs_node {
149 	struct list_head	list;
150 	struct list_head	children;
151 	enum fs_node_type	type;
152 	struct fs_node		*parent;
153 	struct fs_node		*root;
154 	/* lock the node for writing and traversing */
155 	struct rw_semaphore	lock;
156 	refcount_t		refcount;
157 	bool			active;
158 	void			(*del_hw_func)(struct fs_node *);
159 	void			(*del_sw_func)(struct fs_node *);
160 	atomic_t		version;
161 };
162 
163 struct mlx5_flow_rule {
164 	struct fs_node				node;
165 	struct mlx5_flow_table			*ft;
166 	struct mlx5_flow_destination		dest_attr;
167 	/* next_ft should be accessed under chain_lock and only of
168 	 * destination type is FWD_NEXT_fT.
169 	 */
170 	struct list_head			next_ft;
171 	u32					sw_action;
172 };
173 
174 struct mlx5_flow_handle {
175 	int num_rules;
176 	struct mlx5_flow_rule *rule[];
177 };
178 
179 /* Type of children is mlx5_flow_group */
180 struct mlx5_flow_table {
181 	struct fs_node			node;
182 	struct mlx5_fs_dr_table		fs_dr_table;
183 	u32				id;
184 	u16				vport;
185 	unsigned int			max_fte;
186 	unsigned int			level;
187 	enum fs_flow_table_type		type;
188 	enum fs_flow_table_op_mod	op_mod;
189 	struct {
190 		bool			active;
191 		unsigned int		required_groups;
192 		unsigned int		group_size;
193 		unsigned int		num_groups;
194 		unsigned int		max_fte;
195 	} autogroup;
196 	/* Protect fwd_rules */
197 	struct mutex			lock;
198 	/* FWD rules that point on this flow table */
199 	struct list_head		fwd_rules;
200 	u32				flags;
201 	struct rhltable			fgs_hash;
202 	enum mlx5_flow_table_miss_action def_miss_action;
203 	struct mlx5_flow_namespace	*ns;
204 };
205 
206 struct mlx5_ft_underlay_qp {
207 	struct list_head list;
208 	u32 qpn;
209 };
210 
211 #define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_e00
212 /* Calculate the fte_match_param length and without the reserved length.
213  * Make sure the reserved field is the last.
214  */
215 #define MLX5_ST_SZ_DW_MATCH_PARAM					    \
216 	((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
217 	 BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) !=		     \
218 			   MLX5_FLD_SZ_BYTES(fte_match_param,		     \
219 					     MLX5_FTE_MATCH_PARAM_RESERVED) +\
220 			   MLX5_BYTE_OFF(fte_match_param,		     \
221 					 MLX5_FTE_MATCH_PARAM_RESERVED)))
222 
223 /* Type of children is mlx5_flow_rule */
224 struct fs_fte {
225 	struct fs_node			node;
226 	struct mlx5_fs_dr_rule		fs_dr_rule;
227 	u32				val[MLX5_ST_SZ_DW_MATCH_PARAM];
228 	u32				dests_size;
229 	u32				fwd_dests;
230 	u32				index;
231 	struct mlx5_flow_context	flow_context;
232 	struct mlx5_flow_act		action;
233 	enum fs_fte_status		status;
234 	struct mlx5_fc			*counter;
235 	struct rhash_head		hash;
236 	int				modify_mask;
237 };
238 
239 /* Type of children is mlx5_flow_table/namespace */
240 struct fs_prio {
241 	struct fs_node			node;
242 	unsigned int			num_levels;
243 	unsigned int			start_level;
244 	unsigned int			prio;
245 	unsigned int			num_ft;
246 };
247 
248 /* Type of children is fs_prio */
249 struct mlx5_flow_namespace {
250 	/* parent == NULL => root ns */
251 	struct	fs_node			node;
252 	enum mlx5_flow_table_miss_action def_miss_action;
253 };
254 
255 struct mlx5_flow_group_mask {
256 	u8	match_criteria_enable;
257 	u32	match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
258 };
259 
260 /* Type of children is fs_fte */
261 struct mlx5_flow_group {
262 	struct fs_node			node;
263 	struct mlx5_fs_dr_matcher	fs_dr_matcher;
264 	struct mlx5_flow_group_mask	mask;
265 	u32				start_index;
266 	u32				max_ftes;
267 	struct ida			fte_allocator;
268 	u32				id;
269 	struct rhashtable		ftes_hash;
270 	struct rhlist_head		hash;
271 };
272 
273 struct mlx5_flow_root_namespace {
274 	struct mlx5_flow_namespace	ns;
275 	enum   mlx5_flow_steering_mode	mode;
276 	struct mlx5_fs_dr_domain	fs_dr_domain;
277 	enum   fs_flow_table_type	table_type;
278 	struct mlx5_core_dev		*dev;
279 	struct mlx5_flow_table		*root_ft;
280 	/* Should be held when chaining flow tables */
281 	struct mutex			chain_lock;
282 	struct list_head		underlay_qpns;
283 	const struct mlx5_flow_cmds	*cmds;
284 };
285 
286 int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
287 void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
288 void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
289 			      struct delayed_work *dwork,
290 			      unsigned long delay);
291 void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
292 				      unsigned long interval);
293 
294 const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
295 
296 int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
297 				 struct mlx5_flow_root_namespace *peer_ns);
298 
299 int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
300 				 enum mlx5_flow_steering_mode mode);
301 
302 int mlx5_fs_core_alloc(struct mlx5_core_dev *dev);
303 void mlx5_fs_core_free(struct mlx5_core_dev *dev);
304 int mlx5_fs_core_init(struct mlx5_core_dev *dev);
305 void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev);
306 
307 int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports);
308 void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev);
309 int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports);
310 void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev);
311 
312 u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type);
313 
314 struct mlx5_flow_root_namespace *find_root(struct fs_node *node);
315 
316 #define fs_get_obj(v, _node)  {v = container_of((_node), typeof(*v), node); }
317 
318 #define fs_list_for_each_entry(pos, root)		\
319 	list_for_each_entry(pos, root, node.list)
320 
321 #define fs_list_for_each_entry_safe(pos, tmp, root)		\
322 	list_for_each_entry_safe(pos, tmp, root, node.list)
323 
324 #define fs_for_each_ns_or_ft_reverse(pos, prio)				\
325 	list_for_each_entry_reverse(pos, &(prio)->node.children, list)
326 
327 #define fs_for_each_ns_or_ft(pos, prio)					\
328 	list_for_each_entry(pos, (&(prio)->node.children), list)
329 
330 #define fs_for_each_prio(pos, ns)			\
331 	fs_list_for_each_entry(pos, &(ns)->node.children)
332 
333 #define fs_for_each_ns(pos, prio)			\
334 	fs_list_for_each_entry(pos, &(prio)->node.children)
335 
336 #define fs_for_each_ft(pos, prio)			\
337 	fs_list_for_each_entry(pos, &(prio)->node.children)
338 
339 #define fs_for_each_ft_safe(pos, tmp, prio)			\
340 	fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children)
341 
342 #define fs_for_each_fg(pos, ft)			\
343 	fs_list_for_each_entry(pos, &(ft)->node.children)
344 
345 #define fs_for_each_fte(pos, fg)			\
346 	fs_list_for_each_entry(pos, &(fg)->node.children)
347 
348 #define fs_for_each_dst(pos, fte)			\
349 	fs_list_for_each_entry(pos, &(fte)->node.children)
350 
351 #define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) (		\
352 	(type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) :		\
353 	(type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) :		\
354 	(type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) :		\
355 	(type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) :		\
356 	(type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) :		\
357 	(type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) :		\
358 	(type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) :		\
359 	(type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) :		\
360 	(type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) :      \
361 	(type == FS_FT_PORT_SEL) ? MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) :      \
362 	(BUILD_BUG_ON_ZERO(FS_FT_PORT_SEL != FS_FT_MAX_TYPE))\
363 	)
364 
365 #endif
366