1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2019 Mellanox Technologies.
3 
4 #include "en_accel/ktls.h"
5 #include "en_accel/ktls_txrx.h"
6 #include "en_accel/ktls_utils.h"
7 
8 struct mlx5e_dump_wqe {
9 	struct mlx5_wqe_ctrl_seg ctrl;
10 	struct mlx5_wqe_data_seg data;
11 };
12 
13 #define MLX5E_KTLS_DUMP_WQEBBS \
14 	(DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB))
15 
16 static u8
17 mlx5e_ktls_dumps_num_wqes(struct mlx5e_params *params, unsigned int nfrags,
18 			  unsigned int sync_len)
19 {
20 	/* Given the MTU and sync_len, calculates an upper bound for the
21 	 * number of DUMP WQEs needed for the TX resync of a record.
22 	 */
23 	return nfrags + DIV_ROUND_UP(sync_len, MLX5E_SW2HW_MTU(params, params->sw_mtu));
24 }
25 
26 u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
27 {
28 	u16 num_dumps, stop_room = 0;
29 
30 	if (!mlx5e_is_ktls_tx(mdev))
31 		return 0;
32 
33 	num_dumps = mlx5e_ktls_dumps_num_wqes(params, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE);
34 
35 	stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS);
36 	stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS);
37 	stop_room += num_dumps * mlx5e_stop_room_for_wqe(mdev, MLX5E_KTLS_DUMP_WQEBBS);
38 
39 	return stop_room;
40 }
41 
42 static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn)
43 {
44 	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
45 	void *tisc;
46 
47 	tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
48 
49 	MLX5_SET(tisc, tisc, tls_en, 1);
50 
51 	return mlx5e_create_tis(mdev, in, tisn);
52 }
53 
54 struct mlx5e_ktls_offload_context_tx {
55 	struct tls_offload_context_tx *tx_ctx;
56 	struct tls12_crypto_info_aes_gcm_128 crypto_info;
57 	struct mlx5e_tls_sw_stats *sw_stats;
58 	u32 expected_seq;
59 	u32 tisn;
60 	u32 key_id;
61 	bool ctx_post_pending;
62 };
63 
64 static void
65 mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx,
66 			   struct mlx5e_ktls_offload_context_tx *priv_tx)
67 {
68 	struct mlx5e_ktls_offload_context_tx **ctx =
69 		__tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX);
70 
71 	BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_tx *) >
72 		     TLS_OFFLOAD_CONTEXT_SIZE_TX);
73 
74 	*ctx = priv_tx;
75 }
76 
77 static struct mlx5e_ktls_offload_context_tx *
78 mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx)
79 {
80 	struct mlx5e_ktls_offload_context_tx **ctx =
81 		__tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX);
82 
83 	return *ctx;
84 }
85 
86 int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
87 		      struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn)
88 {
89 	struct mlx5e_ktls_offload_context_tx *priv_tx;
90 	struct tls_context *tls_ctx;
91 	struct mlx5_core_dev *mdev;
92 	struct mlx5e_priv *priv;
93 	int err;
94 
95 	tls_ctx = tls_get_ctx(sk);
96 	priv = netdev_priv(netdev);
97 	mdev = priv->mdev;
98 
99 	priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL);
100 	if (!priv_tx)
101 		return -ENOMEM;
102 
103 	err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id);
104 	if (err)
105 		goto err_create_key;
106 
107 	priv_tx->sw_stats = &priv->tls->sw_stats;
108 	priv_tx->expected_seq = start_offload_tcp_sn;
109 	priv_tx->crypto_info  =
110 		*(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
111 	priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx);
112 
113 	mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx);
114 
115 	err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn);
116 	if (err)
117 		goto err_create_tis;
118 
119 	priv_tx->ctx_post_pending = true;
120 	atomic64_inc(&priv_tx->sw_stats->tx_tls_ctx);
121 
122 	return 0;
123 
124 err_create_tis:
125 	mlx5_ktls_destroy_key(mdev, priv_tx->key_id);
126 err_create_key:
127 	kfree(priv_tx);
128 	return err;
129 }
130 
131 void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx)
132 {
133 	struct mlx5e_ktls_offload_context_tx *priv_tx;
134 	struct mlx5_core_dev *mdev;
135 	struct mlx5e_priv *priv;
136 
137 	priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
138 	priv = netdev_priv(netdev);
139 	mdev = priv->mdev;
140 
141 	atomic64_inc(&priv_tx->sw_stats->tx_tls_del);
142 	mlx5e_destroy_tis(mdev, priv_tx->tisn);
143 	mlx5_ktls_destroy_key(mdev, priv_tx->key_id);
144 	kfree(priv_tx);
145 }
146 
147 static void tx_fill_wi(struct mlx5e_txqsq *sq,
148 		       u16 pi, u8 num_wqebbs, u32 num_bytes,
149 		       struct page *page)
150 {
151 	struct mlx5e_tx_wqe_info *wi = &sq->db.wqe_info[pi];
152 
153 	*wi = (struct mlx5e_tx_wqe_info) {
154 		.num_wqebbs = num_wqebbs,
155 		.num_bytes  = num_bytes,
156 		.resync_dump_frag_page = page,
157 	};
158 }
159 
160 static bool
161 mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
162 {
163 	bool ret = priv_tx->ctx_post_pending;
164 
165 	priv_tx->ctx_post_pending = false;
166 
167 	return ret;
168 }
169 
170 static void
171 post_static_params(struct mlx5e_txqsq *sq,
172 		   struct mlx5e_ktls_offload_context_tx *priv_tx,
173 		   bool fence)
174 {
175 	struct mlx5e_set_tls_static_params_wqe *wqe;
176 	u16 pi, num_wqebbs;
177 
178 	num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
179 	pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
180 	wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
181 	mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info,
182 				       priv_tx->tisn, priv_tx->key_id, 0, fence,
183 				       TLS_OFFLOAD_CTX_DIR_TX);
184 	tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
185 	sq->pc += num_wqebbs;
186 }
187 
188 static void
189 post_progress_params(struct mlx5e_txqsq *sq,
190 		     struct mlx5e_ktls_offload_context_tx *priv_tx,
191 		     bool fence)
192 {
193 	struct mlx5e_set_tls_progress_params_wqe *wqe;
194 	u16 pi, num_wqebbs;
195 
196 	num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
197 	pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
198 	wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
199 	mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, 0,
200 					 TLS_OFFLOAD_CTX_DIR_TX);
201 	tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
202 	sq->pc += num_wqebbs;
203 }
204 
205 static void
206 mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq,
207 			      struct mlx5e_ktls_offload_context_tx *priv_tx,
208 			      bool skip_static_post, bool fence_first_post)
209 {
210 	bool progress_fence = skip_static_post || !fence_first_post;
211 
212 	if (!skip_static_post)
213 		post_static_params(sq, priv_tx, fence_first_post);
214 
215 	post_progress_params(sq, priv_tx, progress_fence);
216 }
217 
218 struct tx_sync_info {
219 	u64 rcd_sn;
220 	u32 sync_len;
221 	int nr_frags;
222 	skb_frag_t frags[MAX_SKB_FRAGS];
223 };
224 
225 enum mlx5e_ktls_sync_retval {
226 	MLX5E_KTLS_SYNC_DONE,
227 	MLX5E_KTLS_SYNC_FAIL,
228 	MLX5E_KTLS_SYNC_SKIP_NO_DATA,
229 };
230 
231 static enum mlx5e_ktls_sync_retval
232 tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx,
233 		 u32 tcp_seq, int datalen, struct tx_sync_info *info)
234 {
235 	struct tls_offload_context_tx *tx_ctx = priv_tx->tx_ctx;
236 	enum mlx5e_ktls_sync_retval ret = MLX5E_KTLS_SYNC_DONE;
237 	struct tls_record_info *record;
238 	int remaining, i = 0;
239 	unsigned long flags;
240 	bool ends_before;
241 
242 	spin_lock_irqsave(&tx_ctx->lock, flags);
243 	record = tls_get_record(tx_ctx, tcp_seq, &info->rcd_sn);
244 
245 	if (unlikely(!record)) {
246 		ret = MLX5E_KTLS_SYNC_FAIL;
247 		goto out;
248 	}
249 
250 	/* There are the following cases:
251 	 * 1. packet ends before start marker: bypass offload.
252 	 * 2. packet starts before start marker and ends after it: drop,
253 	 *    not supported, breaks contract with kernel.
254 	 * 3. packet ends before tls record info starts: drop,
255 	 *    this packet was already acknowledged and its record info
256 	 *    was released.
257 	 */
258 	ends_before = before(tcp_seq + datalen - 1, tls_record_start_seq(record));
259 
260 	if (unlikely(tls_record_is_start_marker(record))) {
261 		ret = ends_before ? MLX5E_KTLS_SYNC_SKIP_NO_DATA : MLX5E_KTLS_SYNC_FAIL;
262 		goto out;
263 	} else if (ends_before) {
264 		ret = MLX5E_KTLS_SYNC_FAIL;
265 		goto out;
266 	}
267 
268 	info->sync_len = tcp_seq - tls_record_start_seq(record);
269 	remaining = info->sync_len;
270 	while (remaining > 0) {
271 		skb_frag_t *frag = &record->frags[i];
272 
273 		get_page(skb_frag_page(frag));
274 		remaining -= skb_frag_size(frag);
275 		info->frags[i++] = *frag;
276 	}
277 	/* reduce the part which will be sent with the original SKB */
278 	if (remaining < 0)
279 		skb_frag_size_add(&info->frags[i - 1], remaining);
280 	info->nr_frags = i;
281 out:
282 	spin_unlock_irqrestore(&tx_ctx->lock, flags);
283 	return ret;
284 }
285 
286 static void
287 tx_post_resync_params(struct mlx5e_txqsq *sq,
288 		      struct mlx5e_ktls_offload_context_tx *priv_tx,
289 		      u64 rcd_sn)
290 {
291 	struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
292 	__be64 rn_be = cpu_to_be64(rcd_sn);
293 	bool skip_static_post;
294 	u16 rec_seq_sz;
295 	char *rec_seq;
296 
297 	rec_seq = info->rec_seq;
298 	rec_seq_sz = sizeof(info->rec_seq);
299 
300 	skip_static_post = !memcmp(rec_seq, &rn_be, rec_seq_sz);
301 	if (!skip_static_post)
302 		memcpy(rec_seq, &rn_be, rec_seq_sz);
303 
304 	mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, skip_static_post, true);
305 }
306 
307 static int
308 tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool first)
309 {
310 	struct mlx5_wqe_ctrl_seg *cseg;
311 	struct mlx5_wqe_data_seg *dseg;
312 	struct mlx5e_dump_wqe *wqe;
313 	dma_addr_t dma_addr = 0;
314 	u16 ds_cnt;
315 	int fsz;
316 	u16 pi;
317 
318 	BUILD_BUG_ON(MLX5E_KTLS_DUMP_WQEBBS != 1);
319 	pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
320 	wqe = MLX5E_TLS_FETCH_DUMP_WQE(sq, pi);
321 
322 	ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
323 
324 	cseg = &wqe->ctrl;
325 	dseg = &wqe->data;
326 
327 	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8)  | MLX5_OPCODE_DUMP);
328 	cseg->qpn_ds           = cpu_to_be32((sq->sqn << 8) | ds_cnt);
329 	cseg->tis_tir_num      = cpu_to_be32(tisn << 8);
330 	cseg->fm_ce_se         = first ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
331 
332 	fsz = skb_frag_size(frag);
333 	dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
334 				    DMA_TO_DEVICE);
335 	if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
336 		return -ENOMEM;
337 
338 	dseg->addr       = cpu_to_be64(dma_addr);
339 	dseg->lkey       = sq->mkey_be;
340 	dseg->byte_count = cpu_to_be32(fsz);
341 	mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE);
342 
343 	tx_fill_wi(sq, pi, MLX5E_KTLS_DUMP_WQEBBS, fsz, skb_frag_page(frag));
344 	sq->pc += MLX5E_KTLS_DUMP_WQEBBS;
345 
346 	return 0;
347 }
348 
349 void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
350 					   struct mlx5e_tx_wqe_info *wi,
351 					   u32 *dma_fifo_cc)
352 {
353 	struct mlx5e_sq_stats *stats;
354 	struct mlx5e_sq_dma *dma;
355 
356 	dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++);
357 	stats = sq->stats;
358 
359 	mlx5e_tx_dma_unmap(sq->pdev, dma);
360 	put_page(wi->resync_dump_frag_page);
361 	stats->tls_dump_packets++;
362 	stats->tls_dump_bytes += wi->num_bytes;
363 }
364 
365 static void tx_post_fence_nop(struct mlx5e_txqsq *sq)
366 {
367 	struct mlx5_wq_cyc *wq = &sq->wq;
368 	u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
369 
370 	tx_fill_wi(sq, pi, 1, 0, NULL);
371 
372 	mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc);
373 }
374 
375 static enum mlx5e_ktls_sync_retval
376 mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx,
377 			 struct mlx5e_txqsq *sq,
378 			 int datalen,
379 			 u32 seq)
380 {
381 	struct mlx5e_sq_stats *stats = sq->stats;
382 	enum mlx5e_ktls_sync_retval ret;
383 	struct tx_sync_info info = {};
384 	int i = 0;
385 
386 	ret = tx_sync_info_get(priv_tx, seq, datalen, &info);
387 	if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) {
388 		if (ret == MLX5E_KTLS_SYNC_SKIP_NO_DATA) {
389 			stats->tls_skip_no_sync_data++;
390 			return MLX5E_KTLS_SYNC_SKIP_NO_DATA;
391 		}
392 		/* We might get here if a retransmission reaches the driver
393 		 * after the relevant record is acked.
394 		 * It should be safe to drop the packet in this case
395 		 */
396 		stats->tls_drop_no_sync_data++;
397 		goto err_out;
398 	}
399 
400 	stats->tls_ooo++;
401 
402 	tx_post_resync_params(sq, priv_tx, info.rcd_sn);
403 
404 	/* If no dump WQE was sent, we need to have a fence NOP WQE before the
405 	 * actual data xmit.
406 	 */
407 	if (!info.nr_frags) {
408 		tx_post_fence_nop(sq);
409 		return MLX5E_KTLS_SYNC_DONE;
410 	}
411 
412 	for (; i < info.nr_frags; i++) {
413 		unsigned int orig_fsz, frag_offset = 0, n = 0;
414 		skb_frag_t *f = &info.frags[i];
415 
416 		orig_fsz = skb_frag_size(f);
417 
418 		do {
419 			bool fence = !(i || frag_offset);
420 			unsigned int fsz;
421 
422 			n++;
423 			fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset);
424 			skb_frag_size_set(f, fsz);
425 			if (tx_post_resync_dump(sq, f, priv_tx->tisn, fence)) {
426 				page_ref_add(skb_frag_page(f), n - 1);
427 				goto err_out;
428 			}
429 
430 			skb_frag_off_add(f, fsz);
431 			frag_offset += fsz;
432 		} while (frag_offset < orig_fsz);
433 
434 		page_ref_add(skb_frag_page(f), n - 1);
435 	}
436 
437 	return MLX5E_KTLS_SYNC_DONE;
438 
439 err_out:
440 	for (; i < info.nr_frags; i++)
441 		/* The put_page() here undoes the page ref obtained in tx_sync_info_get().
442 		 * Page refs obtained for the DUMP WQEs above (by page_ref_add) will be
443 		 * released only upon their completions (or in mlx5e_free_txqsq_descs,
444 		 * if channel closes).
445 		 */
446 		put_page(skb_frag_page(&info.frags[i]));
447 
448 	return MLX5E_KTLS_SYNC_FAIL;
449 }
450 
451 bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq,
452 			      struct sk_buff *skb,
453 			      struct mlx5e_accel_tx_tls_state *state)
454 {
455 	struct mlx5e_ktls_offload_context_tx *priv_tx;
456 	struct mlx5e_sq_stats *stats = sq->stats;
457 	struct tls_context *tls_ctx;
458 	int datalen;
459 	u32 seq;
460 
461 	datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb));
462 	if (!datalen)
463 		return true;
464 
465 	mlx5e_tx_mpwqe_ensure_complete(sq);
466 
467 	tls_ctx = tls_get_ctx(skb->sk);
468 	if (WARN_ON_ONCE(tls_ctx->netdev != netdev))
469 		goto err_out;
470 
471 	priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
472 
473 	if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) {
474 		mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false);
475 	}
476 
477 	seq = ntohl(tcp_hdr(skb)->seq);
478 	if (unlikely(priv_tx->expected_seq != seq)) {
479 		enum mlx5e_ktls_sync_retval ret =
480 			mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq);
481 
482 		switch (ret) {
483 		case MLX5E_KTLS_SYNC_DONE:
484 			break;
485 		case MLX5E_KTLS_SYNC_SKIP_NO_DATA:
486 			if (likely(!skb->decrypted))
487 				goto out;
488 			WARN_ON_ONCE(1);
489 			fallthrough;
490 		case MLX5E_KTLS_SYNC_FAIL:
491 			goto err_out;
492 		}
493 	}
494 
495 	priv_tx->expected_seq = seq + datalen;
496 
497 	state->tls_tisn = priv_tx->tisn;
498 
499 	stats->tls_encrypted_packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
500 	stats->tls_encrypted_bytes   += datalen;
501 
502 out:
503 	return true;
504 
505 err_out:
506 	dev_kfree_skb_any(skb);
507 	return false;
508 }
509