1 /*
2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <crypto/aead.h>
35 #include <net/xfrm.h>
36 #include <net/esp.h>
37 #include "ipsec.h"
38 #include "ipsec_rxtx.h"
39 #include "en.h"
40 #include "esw/ipsec_fs.h"
41
42 enum {
43 MLX5E_IPSEC_TX_SYNDROME_OFFLOAD = 0x8,
44 MLX5E_IPSEC_TX_SYNDROME_OFFLOAD_WITH_LSO_TCP = 0x9,
45 };
46
mlx5e_ipsec_remove_trailer(struct sk_buff * skb,struct xfrm_state * x)47 static int mlx5e_ipsec_remove_trailer(struct sk_buff *skb, struct xfrm_state *x)
48 {
49 unsigned int alen = crypto_aead_authsize(x->data);
50 struct ipv6hdr *ipv6hdr = ipv6_hdr(skb);
51 struct iphdr *ipv4hdr = ip_hdr(skb);
52 unsigned int trailer_len;
53 u8 plen;
54 int ret;
55
56 ret = skb_copy_bits(skb, skb->len - alen - 2, &plen, 1);
57 if (unlikely(ret))
58 return ret;
59
60 trailer_len = alen + plen + 2;
61
62 ret = pskb_trim(skb, skb->len - trailer_len);
63 if (unlikely(ret))
64 return ret;
65 if (skb->protocol == htons(ETH_P_IP)) {
66 ipv4hdr->tot_len = htons(ntohs(ipv4hdr->tot_len) - trailer_len);
67 ip_send_check(ipv4hdr);
68 } else {
69 ipv6hdr->payload_len = htons(ntohs(ipv6hdr->payload_len) -
70 trailer_len);
71 }
72 return 0;
73 }
74
mlx5e_ipsec_set_swp(struct sk_buff * skb,struct mlx5_wqe_eth_seg * eseg,u8 mode,struct xfrm_offload * xo)75 static void mlx5e_ipsec_set_swp(struct sk_buff *skb,
76 struct mlx5_wqe_eth_seg *eseg, u8 mode,
77 struct xfrm_offload *xo)
78 {
79 /* Tunnel Mode:
80 * SWP: OutL3 InL3 InL4
81 * Pkt: MAC IP ESP IP L4
82 *
83 * Transport Mode:
84 * SWP: OutL3 OutL4
85 * Pkt: MAC IP ESP L4
86 *
87 * Tunnel(VXLAN TCP/UDP) over Transport Mode
88 * SWP: OutL3 InL3 InL4
89 * Pkt: MAC IP ESP UDP VXLAN IP L4
90 */
91
92 /* Shared settings */
93 eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2;
94 if (skb->protocol == htons(ETH_P_IPV6))
95 eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6;
96
97 /* Tunnel mode */
98 if (mode == XFRM_MODE_TUNNEL) {
99 eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
100 if (xo->proto == IPPROTO_IPV6)
101 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
102
103 switch (xo->inner_ipproto) {
104 case IPPROTO_UDP:
105 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
106 fallthrough;
107 case IPPROTO_TCP:
108 /* IP | ESP | IP | [TCP | UDP] */
109 eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
110 break;
111 default:
112 break;
113 }
114 return;
115 }
116
117 /* Transport mode */
118 if (mode != XFRM_MODE_TRANSPORT)
119 return;
120
121 if (!xo->inner_ipproto) {
122 switch (xo->proto) {
123 case IPPROTO_UDP:
124 eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP;
125 fallthrough;
126 case IPPROTO_TCP:
127 /* IP | ESP | TCP */
128 eseg->swp_outer_l4_offset = skb_inner_transport_offset(skb) / 2;
129 break;
130 default:
131 break;
132 }
133 } else {
134 /* Tunnel(VXLAN TCP/UDP) over Transport Mode */
135 switch (xo->inner_ipproto) {
136 case IPPROTO_UDP:
137 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
138 fallthrough;
139 case IPPROTO_TCP:
140 eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
141 eseg->swp_inner_l4_offset =
142 (skb->csum_start + skb->head - skb->data) / 2;
143 if (inner_ip_hdr(skb)->version == 6)
144 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
145 break;
146 default:
147 break;
148 }
149 }
150
151 }
152
mlx5e_ipsec_set_iv_esn(struct sk_buff * skb,struct xfrm_state * x,struct xfrm_offload * xo)153 void mlx5e_ipsec_set_iv_esn(struct sk_buff *skb, struct xfrm_state *x,
154 struct xfrm_offload *xo)
155 {
156 struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
157 __u32 oseq = replay_esn->oseq;
158 int iv_offset;
159 __be64 seqno;
160 u32 seq_hi;
161
162 if (unlikely(skb_is_gso(skb) && oseq < MLX5E_IPSEC_ESN_SCOPE_MID &&
163 MLX5E_IPSEC_ESN_SCOPE_MID < (oseq - skb_shinfo(skb)->gso_segs))) {
164 seq_hi = xo->seq.hi - 1;
165 } else {
166 seq_hi = xo->seq.hi;
167 }
168
169 /* Place the SN in the IV field */
170 seqno = cpu_to_be64(xo->seq.low + ((u64)seq_hi << 32));
171 iv_offset = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr);
172 skb_store_bits(skb, iv_offset, &seqno, 8);
173 }
174
mlx5e_ipsec_set_iv(struct sk_buff * skb,struct xfrm_state * x,struct xfrm_offload * xo)175 void mlx5e_ipsec_set_iv(struct sk_buff *skb, struct xfrm_state *x,
176 struct xfrm_offload *xo)
177 {
178 int iv_offset;
179 __be64 seqno;
180
181 /* Place the SN in the IV field */
182 seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));
183 iv_offset = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr);
184 skb_store_bits(skb, iv_offset, &seqno, 8);
185 }
186
mlx5e_ipsec_handle_tx_wqe(struct mlx5e_tx_wqe * wqe,struct mlx5e_accel_tx_ipsec_state * ipsec_st,struct mlx5_wqe_inline_seg * inlseg)187 void mlx5e_ipsec_handle_tx_wqe(struct mlx5e_tx_wqe *wqe,
188 struct mlx5e_accel_tx_ipsec_state *ipsec_st,
189 struct mlx5_wqe_inline_seg *inlseg)
190 {
191 inlseg->byte_count = cpu_to_be32(ipsec_st->tailen | MLX5_INLINE_SEG);
192 esp_output_fill_trailer((u8 *)inlseg->data, 0, ipsec_st->plen, ipsec_st->xo->proto);
193 }
194
mlx5e_ipsec_set_state(struct mlx5e_priv * priv,struct sk_buff * skb,struct xfrm_state * x,struct xfrm_offload * xo,struct mlx5e_accel_tx_ipsec_state * ipsec_st)195 static int mlx5e_ipsec_set_state(struct mlx5e_priv *priv,
196 struct sk_buff *skb,
197 struct xfrm_state *x,
198 struct xfrm_offload *xo,
199 struct mlx5e_accel_tx_ipsec_state *ipsec_st)
200 {
201 unsigned int blksize, clen, alen, plen;
202 struct crypto_aead *aead;
203 unsigned int tailen;
204
205 ipsec_st->x = x;
206 ipsec_st->xo = xo;
207 aead = x->data;
208 alen = crypto_aead_authsize(aead);
209 blksize = ALIGN(crypto_aead_blocksize(aead), 4);
210 clen = ALIGN(skb->len + 2, blksize);
211 plen = max_t(u32, clen - skb->len, 4);
212 tailen = plen + alen;
213 ipsec_st->plen = plen;
214 ipsec_st->tailen = tailen;
215
216 return 0;
217 }
218
mlx5e_ipsec_tx_build_eseg(struct mlx5e_priv * priv,struct sk_buff * skb,struct mlx5_wqe_eth_seg * eseg)219 void mlx5e_ipsec_tx_build_eseg(struct mlx5e_priv *priv, struct sk_buff *skb,
220 struct mlx5_wqe_eth_seg *eseg)
221 {
222 struct xfrm_offload *xo = xfrm_offload(skb);
223 struct xfrm_encap_tmpl *encap;
224 struct xfrm_state *x;
225 struct sec_path *sp;
226 u8 l3_proto;
227
228 sp = skb_sec_path(skb);
229 if (unlikely(sp->len != 1))
230 return;
231
232 x = xfrm_input_state(skb);
233 if (unlikely(!x))
234 return;
235
236 if (unlikely(!x->xso.offload_handle ||
237 (skb->protocol != htons(ETH_P_IP) &&
238 skb->protocol != htons(ETH_P_IPV6))))
239 return;
240
241 mlx5e_ipsec_set_swp(skb, eseg, x->props.mode, xo);
242
243 l3_proto = (x->props.family == AF_INET) ?
244 ((struct iphdr *)skb_network_header(skb))->protocol :
245 ((struct ipv6hdr *)skb_network_header(skb))->nexthdr;
246
247 eseg->flow_table_metadata |= cpu_to_be32(MLX5_ETH_WQE_FT_META_IPSEC);
248 eseg->trailer |= cpu_to_be32(MLX5_ETH_WQE_INSERT_TRAILER);
249 encap = x->encap;
250 if (!encap) {
251 eseg->trailer |= (l3_proto == IPPROTO_ESP) ?
252 cpu_to_be32(MLX5_ETH_WQE_TRAILER_HDR_OUTER_IP_ASSOC) :
253 cpu_to_be32(MLX5_ETH_WQE_TRAILER_HDR_OUTER_L4_ASSOC);
254 } else if (encap->encap_type == UDP_ENCAP_ESPINUDP) {
255 eseg->trailer |= (l3_proto == IPPROTO_ESP) ?
256 cpu_to_be32(MLX5_ETH_WQE_TRAILER_HDR_INNER_IP_ASSOC) :
257 cpu_to_be32(MLX5_ETH_WQE_TRAILER_HDR_INNER_L4_ASSOC);
258 }
259 }
260
mlx5e_ipsec_handle_tx_skb(struct net_device * netdev,struct sk_buff * skb,struct mlx5e_accel_tx_ipsec_state * ipsec_st)261 bool mlx5e_ipsec_handle_tx_skb(struct net_device *netdev,
262 struct sk_buff *skb,
263 struct mlx5e_accel_tx_ipsec_state *ipsec_st)
264 {
265 struct mlx5e_priv *priv = netdev_priv(netdev);
266 struct xfrm_offload *xo = xfrm_offload(skb);
267 struct mlx5e_ipsec_sa_entry *sa_entry;
268 struct xfrm_state *x;
269 struct sec_path *sp;
270
271 sp = skb_sec_path(skb);
272 if (unlikely(sp->len != 1)) {
273 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_bundle);
274 goto drop;
275 }
276
277 x = xfrm_input_state(skb);
278 if (unlikely(!x)) {
279 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_no_state);
280 goto drop;
281 }
282
283 if (unlikely(!x->xso.offload_handle ||
284 (skb->protocol != htons(ETH_P_IP) &&
285 skb->protocol != htons(ETH_P_IPV6)))) {
286 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_not_ip);
287 goto drop;
288 }
289
290 if (!skb_is_gso(skb))
291 if (unlikely(mlx5e_ipsec_remove_trailer(skb, x))) {
292 atomic64_inc(&priv->ipsec->sw_stats.ipsec_tx_drop_trailer);
293 goto drop;
294 }
295
296 sa_entry = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;
297 sa_entry->set_iv_op(skb, x, xo);
298 mlx5e_ipsec_set_state(priv, skb, x, xo, ipsec_st);
299
300 return true;
301
302 drop:
303 kfree_skb(skb);
304 return false;
305 }
306
307 enum {
308 MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED,
309 MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED,
310 MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER,
311 };
312
mlx5e_ipsec_offload_handle_rx_skb(struct net_device * netdev,struct sk_buff * skb,u32 ipsec_meta_data)313 void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
314 struct sk_buff *skb,
315 u32 ipsec_meta_data)
316 {
317 struct mlx5e_priv *priv = netdev_priv(netdev);
318 struct mlx5e_ipsec *ipsec = priv->ipsec;
319 struct mlx5e_ipsec_sa_entry *sa_entry;
320 struct xfrm_offload *xo;
321 struct sec_path *sp;
322 u32 sa_handle;
323
324 sa_handle = MLX5_IPSEC_METADATA_HANDLE(ipsec_meta_data);
325 sp = secpath_set(skb);
326 if (unlikely(!sp)) {
327 atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sp_alloc);
328 return;
329 }
330
331 rcu_read_lock();
332 sa_entry = xa_load(&ipsec->sadb, sa_handle);
333 if (unlikely(!sa_entry)) {
334 rcu_read_unlock();
335 atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss);
336 return;
337 }
338 xfrm_state_hold(sa_entry->x);
339 rcu_read_unlock();
340
341 sp->xvec[sp->len++] = sa_entry->x;
342 sp->olen++;
343
344 xo = xfrm_offload(skb);
345 xo->flags = CRYPTO_DONE;
346
347 switch (MLX5_IPSEC_METADATA_SYNDROM(ipsec_meta_data)) {
348 case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED:
349 xo->status = CRYPTO_SUCCESS;
350 break;
351 case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED:
352 xo->status = CRYPTO_TUNNEL_ESP_AUTH_FAILED;
353 break;
354 case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER:
355 xo->status = CRYPTO_INVALID_PACKET_SYNTAX;
356 break;
357 default:
358 atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_syndrome);
359 }
360 }
361
mlx5_esw_ipsec_rx_make_metadata(struct mlx5e_priv * priv,u32 id,u32 * metadata)362 int mlx5_esw_ipsec_rx_make_metadata(struct mlx5e_priv *priv, u32 id, u32 *metadata)
363 {
364 struct mlx5e_ipsec *ipsec = priv->ipsec;
365 u32 ipsec_obj_id;
366 int err;
367
368 if (!ipsec || !ipsec->is_uplink_rep)
369 return -EINVAL;
370
371 err = mlx5_esw_ipsec_rx_ipsec_obj_id_search(priv, id, &ipsec_obj_id);
372 if (err) {
373 atomic64_inc(&ipsec->sw_stats.ipsec_rx_drop_sadb_miss);
374 return err;
375 }
376
377 *metadata = MLX5_IPSEC_METADATA_CREATE(ipsec_obj_id,
378 MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED);
379
380 return 0;
381 }
382