1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3 
4 #include "fs_core.h"
5 #include "lib/ipsec_fs_roce.h"
6 #include "mlx5_core.h"
7 
8 struct mlx5_ipsec_miss {
9 	struct mlx5_flow_group *group;
10 	struct mlx5_flow_handle *rule;
11 };
12 
13 struct mlx5_ipsec_rx_roce {
14 	struct mlx5_flow_group *g;
15 	struct mlx5_flow_table *ft;
16 	struct mlx5_flow_handle *rule;
17 	struct mlx5_ipsec_miss roce_miss;
18 
19 	struct mlx5_flow_table *ft_rdma;
20 	struct mlx5_flow_namespace *ns_rdma;
21 };
22 
23 struct mlx5_ipsec_tx_roce {
24 	struct mlx5_flow_group *g;
25 	struct mlx5_flow_table *ft;
26 	struct mlx5_flow_handle *rule;
27 	struct mlx5_flow_namespace *ns;
28 };
29 
30 struct mlx5_ipsec_fs {
31 	struct mlx5_ipsec_rx_roce ipv4_rx;
32 	struct mlx5_ipsec_rx_roce ipv6_rx;
33 	struct mlx5_ipsec_tx_roce tx;
34 };
35 
ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec * spec,u16 dport)36 static void ipsec_fs_roce_setup_udp_dport(struct mlx5_flow_spec *spec,
37 					  u16 dport)
38 {
39 	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
40 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
41 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
42 	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
43 	MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport);
44 }
45 
46 static int
ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev * mdev,struct mlx5_flow_destination * default_dst,struct mlx5_ipsec_rx_roce * roce)47 ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev,
48 			    struct mlx5_flow_destination *default_dst,
49 			    struct mlx5_ipsec_rx_roce *roce)
50 {
51 	struct mlx5_flow_destination dst = {};
52 	MLX5_DECLARE_FLOW_ACT(flow_act);
53 	struct mlx5_flow_handle *rule;
54 	struct mlx5_flow_spec *spec;
55 	int err = 0;
56 
57 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
58 	if (!spec)
59 		return -ENOMEM;
60 
61 	ipsec_fs_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT);
62 
63 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
64 	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
65 	dst.ft = roce->ft_rdma;
66 	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
67 	if (IS_ERR(rule)) {
68 		err = PTR_ERR(rule);
69 		mlx5_core_err(mdev, "Fail to add RX RoCE IPsec rule err=%d\n",
70 			      err);
71 		goto fail_add_rule;
72 	}
73 
74 	roce->rule = rule;
75 
76 	memset(spec, 0, sizeof(*spec));
77 	rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1);
78 	if (IS_ERR(rule)) {
79 		err = PTR_ERR(rule);
80 		mlx5_core_err(mdev, "Fail to add RX RoCE IPsec miss rule err=%d\n",
81 			      err);
82 		goto fail_add_default_rule;
83 	}
84 
85 	roce->roce_miss.rule = rule;
86 
87 	kvfree(spec);
88 	return 0;
89 
90 fail_add_default_rule:
91 	mlx5_del_flow_rules(roce->rule);
92 fail_add_rule:
93 	kvfree(spec);
94 	return err;
95 }
96 
ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev * mdev,struct mlx5_ipsec_tx_roce * roce,struct mlx5_flow_table * pol_ft)97 static int ipsec_fs_roce_tx_rule_setup(struct mlx5_core_dev *mdev,
98 				       struct mlx5_ipsec_tx_roce *roce,
99 				       struct mlx5_flow_table *pol_ft)
100 {
101 	struct mlx5_flow_destination dst = {};
102 	MLX5_DECLARE_FLOW_ACT(flow_act);
103 	struct mlx5_flow_handle *rule;
104 	int err = 0;
105 
106 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
107 	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
108 	dst.ft = pol_ft;
109 	rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, &dst,
110 				   1);
111 	if (IS_ERR(rule)) {
112 		err = PTR_ERR(rule);
113 		mlx5_core_err(mdev, "Fail to add TX RoCE IPsec rule err=%d\n",
114 			      err);
115 		goto out;
116 	}
117 	roce->rule = rule;
118 
119 out:
120 	return err;
121 }
122 
mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs * ipsec_roce)123 void mlx5_ipsec_fs_roce_tx_destroy(struct mlx5_ipsec_fs *ipsec_roce)
124 {
125 	struct mlx5_ipsec_tx_roce *tx_roce;
126 
127 	if (!ipsec_roce)
128 		return;
129 
130 	tx_roce = &ipsec_roce->tx;
131 
132 	mlx5_del_flow_rules(tx_roce->rule);
133 	mlx5_destroy_flow_group(tx_roce->g);
134 	mlx5_destroy_flow_table(tx_roce->ft);
135 }
136 
137 #define MLX5_TX_ROCE_GROUP_SIZE BIT(0)
138 
mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_table * pol_ft)139 int mlx5_ipsec_fs_roce_tx_create(struct mlx5_core_dev *mdev,
140 				 struct mlx5_ipsec_fs *ipsec_roce,
141 				 struct mlx5_flow_table *pol_ft)
142 {
143 	struct mlx5_flow_table_attr ft_attr = {};
144 	struct mlx5_ipsec_tx_roce *roce;
145 	struct mlx5_flow_table *ft;
146 	struct mlx5_flow_group *g;
147 	int ix = 0;
148 	int err;
149 	u32 *in;
150 
151 	if (!ipsec_roce)
152 		return 0;
153 
154 	roce = &ipsec_roce->tx;
155 
156 	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
157 	if (!in)
158 		return -ENOMEM;
159 
160 	ft_attr.max_fte = 1;
161 	ft = mlx5_create_flow_table(roce->ns, &ft_attr);
162 	if (IS_ERR(ft)) {
163 		err = PTR_ERR(ft);
164 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx ft err=%d\n", err);
165 		goto free_in;
166 	}
167 
168 	roce->ft = ft;
169 
170 	MLX5_SET_CFG(in, start_flow_index, ix);
171 	ix += MLX5_TX_ROCE_GROUP_SIZE;
172 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
173 	g = mlx5_create_flow_group(ft, in);
174 	if (IS_ERR(g)) {
175 		err = PTR_ERR(g);
176 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx group err=%d\n", err);
177 		goto destroy_table;
178 	}
179 	roce->g = g;
180 
181 	err = ipsec_fs_roce_tx_rule_setup(mdev, roce, pol_ft);
182 	if (err) {
183 		mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
184 		goto destroy_group;
185 	}
186 
187 	kvfree(in);
188 	return 0;
189 
190 destroy_group:
191 	mlx5_destroy_flow_group(roce->g);
192 destroy_table:
193 	mlx5_destroy_flow_table(ft);
194 free_in:
195 	kvfree(in);
196 	return err;
197 }
198 
mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs * ipsec_roce,u32 family)199 struct mlx5_flow_table *mlx5_ipsec_fs_roce_ft_get(struct mlx5_ipsec_fs *ipsec_roce, u32 family)
200 {
201 	struct mlx5_ipsec_rx_roce *rx_roce;
202 
203 	if (!ipsec_roce)
204 		return NULL;
205 
206 	rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
207 					&ipsec_roce->ipv6_rx;
208 
209 	return rx_roce->ft;
210 }
211 
mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs * ipsec_roce,u32 family)212 void mlx5_ipsec_fs_roce_rx_destroy(struct mlx5_ipsec_fs *ipsec_roce, u32 family)
213 {
214 	struct mlx5_ipsec_rx_roce *rx_roce;
215 
216 	if (!ipsec_roce)
217 		return;
218 
219 	rx_roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
220 					&ipsec_roce->ipv6_rx;
221 
222 	mlx5_del_flow_rules(rx_roce->roce_miss.rule);
223 	mlx5_del_flow_rules(rx_roce->rule);
224 	mlx5_destroy_flow_table(rx_roce->ft_rdma);
225 	mlx5_destroy_flow_group(rx_roce->roce_miss.group);
226 	mlx5_destroy_flow_group(rx_roce->g);
227 	mlx5_destroy_flow_table(rx_roce->ft);
228 }
229 
230 #define MLX5_RX_ROCE_GROUP_SIZE BIT(0)
231 
mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev * mdev,struct mlx5_ipsec_fs * ipsec_roce,struct mlx5_flow_namespace * ns,struct mlx5_flow_destination * default_dst,u32 family,u32 level,u32 prio)232 int mlx5_ipsec_fs_roce_rx_create(struct mlx5_core_dev *mdev,
233 				 struct mlx5_ipsec_fs *ipsec_roce,
234 				 struct mlx5_flow_namespace *ns,
235 				 struct mlx5_flow_destination *default_dst,
236 				 u32 family, u32 level, u32 prio)
237 {
238 	struct mlx5_flow_table_attr ft_attr = {};
239 	struct mlx5_ipsec_rx_roce *roce;
240 	struct mlx5_flow_table *ft;
241 	struct mlx5_flow_group *g;
242 	void *outer_headers_c;
243 	int ix = 0;
244 	u32 *in;
245 	int err;
246 	u8 *mc;
247 
248 	if (!ipsec_roce)
249 		return 0;
250 
251 	roce = (family == AF_INET) ? &ipsec_roce->ipv4_rx :
252 				     &ipsec_roce->ipv6_rx;
253 
254 	ft_attr.max_fte = 2;
255 	ft_attr.level = level;
256 	ft_attr.prio = prio;
257 	ft = mlx5_create_flow_table(ns, &ft_attr);
258 	if (IS_ERR(ft)) {
259 		err = PTR_ERR(ft);
260 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at nic err=%d\n", err);
261 		return err;
262 	}
263 
264 	roce->ft = ft;
265 
266 	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
267 	if (!in) {
268 		err = -ENOMEM;
269 		goto fail_nomem;
270 	}
271 
272 	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
273 	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
274 	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
275 	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
276 
277 	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
278 	MLX5_SET_CFG(in, start_flow_index, ix);
279 	ix += MLX5_RX_ROCE_GROUP_SIZE;
280 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
281 	g = mlx5_create_flow_group(ft, in);
282 	if (IS_ERR(g)) {
283 		err = PTR_ERR(g);
284 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx group at nic err=%d\n", err);
285 		goto fail_group;
286 	}
287 	roce->g = g;
288 
289 	memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in));
290 	MLX5_SET_CFG(in, start_flow_index, ix);
291 	ix += MLX5_RX_ROCE_GROUP_SIZE;
292 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
293 	g = mlx5_create_flow_group(ft, in);
294 	if (IS_ERR(g)) {
295 		err = PTR_ERR(g);
296 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx miss group at nic err=%d\n", err);
297 		goto fail_mgroup;
298 	}
299 	roce->roce_miss.group = g;
300 
301 	memset(&ft_attr, 0, sizeof(ft_attr));
302 	if (family == AF_INET)
303 		ft_attr.level = 1;
304 	ft = mlx5_create_flow_table(roce->ns_rdma, &ft_attr);
305 	if (IS_ERR(ft)) {
306 		err = PTR_ERR(ft);
307 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx ft at rdma err=%d\n", err);
308 		goto fail_rdma_table;
309 	}
310 
311 	roce->ft_rdma = ft;
312 
313 	err = ipsec_fs_roce_rx_rule_setup(mdev, default_dst, roce);
314 	if (err) {
315 		mlx5_core_err(mdev, "Fail to create RoCE IPsec rx rules err=%d\n", err);
316 		goto fail_setup_rule;
317 	}
318 
319 	kvfree(in);
320 	return 0;
321 
322 fail_setup_rule:
323 	mlx5_destroy_flow_table(roce->ft_rdma);
324 fail_rdma_table:
325 	mlx5_destroy_flow_group(roce->roce_miss.group);
326 fail_mgroup:
327 	mlx5_destroy_flow_group(roce->g);
328 fail_group:
329 	kvfree(in);
330 fail_nomem:
331 	mlx5_destroy_flow_table(roce->ft);
332 	return err;
333 }
334 
mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs * ipsec_roce)335 void mlx5_ipsec_fs_roce_cleanup(struct mlx5_ipsec_fs *ipsec_roce)
336 {
337 	kfree(ipsec_roce);
338 }
339 
mlx5_ipsec_fs_roce_init(struct mlx5_core_dev * mdev)340 struct mlx5_ipsec_fs *mlx5_ipsec_fs_roce_init(struct mlx5_core_dev *mdev)
341 {
342 	struct mlx5_ipsec_fs *roce_ipsec;
343 	struct mlx5_flow_namespace *ns;
344 
345 	ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
346 	if (!ns) {
347 		mlx5_core_err(mdev, "Failed to get RoCE rx ns\n");
348 		return NULL;
349 	}
350 
351 	roce_ipsec = kzalloc(sizeof(*roce_ipsec), GFP_KERNEL);
352 	if (!roce_ipsec)
353 		return NULL;
354 
355 	roce_ipsec->ipv4_rx.ns_rdma = ns;
356 	roce_ipsec->ipv6_rx.ns_rdma = ns;
357 
358 	ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
359 	if (!ns) {
360 		mlx5_core_err(mdev, "Failed to get RoCE tx ns\n");
361 		goto err_tx;
362 	}
363 
364 	roce_ipsec->tx.ns = ns;
365 
366 	return roce_ipsec;
367 
368 err_tx:
369 	kfree(roce_ipsec);
370 	return NULL;
371 }
372