1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include "en_accel/ktls.h" 5 #include "en_accel/ktls_txrx.h" 6 #include "en_accel/ktls_utils.h" 7 8 struct mlx5e_dump_wqe { 9 struct mlx5_wqe_ctrl_seg ctrl; 10 struct mlx5_wqe_data_seg data; 11 }; 12 13 #define MLX5E_KTLS_DUMP_WQEBBS \ 14 (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB)) 15 16 static u8 17 mlx5e_ktls_dumps_num_wqes(struct mlx5e_params *params, unsigned int nfrags, 18 unsigned int sync_len) 19 { 20 /* Given the MTU and sync_len, calculates an upper bound for the 21 * number of DUMP WQEs needed for the TX resync of a record. 22 */ 23 return nfrags + DIV_ROUND_UP(sync_len, MLX5E_SW2HW_MTU(params, params->sw_mtu)); 24 } 25 26 u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 27 { 28 u16 num_dumps, stop_room = 0; 29 30 if (!mlx5e_is_ktls_tx(mdev)) 31 return 0; 32 33 num_dumps = mlx5e_ktls_dumps_num_wqes(params, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE); 34 35 stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS); 36 stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS); 37 stop_room += num_dumps * mlx5e_stop_room_for_wqe(mdev, MLX5E_KTLS_DUMP_WQEBBS); 38 39 return stop_room; 40 } 41 42 static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) 43 { 44 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 45 void *tisc; 46 47 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 48 49 MLX5_SET(tisc, tisc, tls_en, 1); 50 51 return mlx5e_create_tis(mdev, in, tisn); 52 } 53 54 struct mlx5e_ktls_offload_context_tx { 55 struct tls_offload_context_tx *tx_ctx; 56 struct tls12_crypto_info_aes_gcm_128 crypto_info; 57 struct mlx5e_tls_sw_stats *sw_stats; 58 u32 expected_seq; 59 u32 tisn; 60 u32 key_id; 61 bool ctx_post_pending; 62 }; 63 64 static void 65 mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx, 66 struct mlx5e_ktls_offload_context_tx *priv_tx) 67 { 68 struct mlx5e_ktls_offload_context_tx **ctx = 69 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX); 70 71 BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_tx *) > 72 TLS_OFFLOAD_CONTEXT_SIZE_TX); 73 74 *ctx = priv_tx; 75 } 76 77 static struct mlx5e_ktls_offload_context_tx * 78 mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) 79 { 80 struct mlx5e_ktls_offload_context_tx **ctx = 81 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX); 82 83 return *ctx; 84 } 85 86 int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, 87 struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn) 88 { 89 struct mlx5e_ktls_offload_context_tx *priv_tx; 90 struct tls_context *tls_ctx; 91 struct mlx5_core_dev *mdev; 92 struct mlx5e_priv *priv; 93 int err; 94 95 tls_ctx = tls_get_ctx(sk); 96 priv = netdev_priv(netdev); 97 mdev = priv->mdev; 98 99 priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL); 100 if (!priv_tx) 101 return -ENOMEM; 102 103 err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id); 104 if (err) 105 goto err_create_key; 106 107 priv_tx->sw_stats = &priv->tls->sw_stats; 108 priv_tx->expected_seq = start_offload_tcp_sn; 109 priv_tx->crypto_info = 110 *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 111 priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx); 112 113 mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx); 114 115 err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn); 116 if (err) 117 goto err_create_tis; 118 119 priv_tx->ctx_post_pending = true; 120 atomic64_inc(&priv_tx->sw_stats->tx_tls_ctx); 121 122 return 0; 123 124 err_create_tis: 125 mlx5_ktls_destroy_key(mdev, priv_tx->key_id); 126 err_create_key: 127 kfree(priv_tx); 128 return err; 129 } 130 131 void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) 132 { 133 struct mlx5e_ktls_offload_context_tx *priv_tx; 134 struct mlx5_core_dev *mdev; 135 struct mlx5e_priv *priv; 136 137 priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); 138 priv = netdev_priv(netdev); 139 mdev = priv->mdev; 140 141 atomic64_inc(&priv_tx->sw_stats->tx_tls_del); 142 mlx5e_destroy_tis(mdev, priv_tx->tisn); 143 mlx5_ktls_destroy_key(mdev, priv_tx->key_id); 144 kfree(priv_tx); 145 } 146 147 static void tx_fill_wi(struct mlx5e_txqsq *sq, 148 u16 pi, u8 num_wqebbs, u32 num_bytes, 149 struct page *page) 150 { 151 struct mlx5e_tx_wqe_info *wi = &sq->db.wqe_info[pi]; 152 153 *wi = (struct mlx5e_tx_wqe_info) { 154 .num_wqebbs = num_wqebbs, 155 .num_bytes = num_bytes, 156 .resync_dump_frag_page = page, 157 }; 158 } 159 160 static bool 161 mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx) 162 { 163 bool ret = priv_tx->ctx_post_pending; 164 165 priv_tx->ctx_post_pending = false; 166 167 return ret; 168 } 169 170 static void 171 post_static_params(struct mlx5e_txqsq *sq, 172 struct mlx5e_ktls_offload_context_tx *priv_tx, 173 bool fence) 174 { 175 struct mlx5e_set_tls_static_params_wqe *wqe; 176 u16 pi, num_wqebbs; 177 178 num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS; 179 pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); 180 wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); 181 mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info, 182 priv_tx->tisn, priv_tx->key_id, 0, fence, 183 TLS_OFFLOAD_CTX_DIR_TX); 184 tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); 185 sq->pc += num_wqebbs; 186 } 187 188 static void 189 post_progress_params(struct mlx5e_txqsq *sq, 190 struct mlx5e_ktls_offload_context_tx *priv_tx, 191 bool fence) 192 { 193 struct mlx5e_set_tls_progress_params_wqe *wqe; 194 u16 pi, num_wqebbs; 195 196 num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; 197 pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); 198 wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); 199 mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, 0, 200 TLS_OFFLOAD_CTX_DIR_TX); 201 tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); 202 sq->pc += num_wqebbs; 203 } 204 205 static void 206 mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq, 207 struct mlx5e_ktls_offload_context_tx *priv_tx, 208 bool skip_static_post, bool fence_first_post) 209 { 210 bool progress_fence = skip_static_post || !fence_first_post; 211 212 if (!skip_static_post) 213 post_static_params(sq, priv_tx, fence_first_post); 214 215 post_progress_params(sq, priv_tx, progress_fence); 216 } 217 218 struct tx_sync_info { 219 u64 rcd_sn; 220 u32 sync_len; 221 int nr_frags; 222 skb_frag_t frags[MAX_SKB_FRAGS]; 223 }; 224 225 enum mlx5e_ktls_sync_retval { 226 MLX5E_KTLS_SYNC_DONE, 227 MLX5E_KTLS_SYNC_FAIL, 228 MLX5E_KTLS_SYNC_SKIP_NO_DATA, 229 }; 230 231 static enum mlx5e_ktls_sync_retval 232 tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx, 233 u32 tcp_seq, int datalen, struct tx_sync_info *info) 234 { 235 struct tls_offload_context_tx *tx_ctx = priv_tx->tx_ctx; 236 enum mlx5e_ktls_sync_retval ret = MLX5E_KTLS_SYNC_DONE; 237 struct tls_record_info *record; 238 int remaining, i = 0; 239 unsigned long flags; 240 bool ends_before; 241 242 spin_lock_irqsave(&tx_ctx->lock, flags); 243 record = tls_get_record(tx_ctx, tcp_seq, &info->rcd_sn); 244 245 if (unlikely(!record)) { 246 ret = MLX5E_KTLS_SYNC_FAIL; 247 goto out; 248 } 249 250 /* There are the following cases: 251 * 1. packet ends before start marker: bypass offload. 252 * 2. packet starts before start marker and ends after it: drop, 253 * not supported, breaks contract with kernel. 254 * 3. packet ends before tls record info starts: drop, 255 * this packet was already acknowledged and its record info 256 * was released. 257 */ 258 ends_before = before(tcp_seq + datalen - 1, tls_record_start_seq(record)); 259 260 if (unlikely(tls_record_is_start_marker(record))) { 261 ret = ends_before ? MLX5E_KTLS_SYNC_SKIP_NO_DATA : MLX5E_KTLS_SYNC_FAIL; 262 goto out; 263 } else if (ends_before) { 264 ret = MLX5E_KTLS_SYNC_FAIL; 265 goto out; 266 } 267 268 info->sync_len = tcp_seq - tls_record_start_seq(record); 269 remaining = info->sync_len; 270 while (remaining > 0) { 271 skb_frag_t *frag = &record->frags[i]; 272 273 get_page(skb_frag_page(frag)); 274 remaining -= skb_frag_size(frag); 275 info->frags[i++] = *frag; 276 } 277 /* reduce the part which will be sent with the original SKB */ 278 if (remaining < 0) 279 skb_frag_size_add(&info->frags[i - 1], remaining); 280 info->nr_frags = i; 281 out: 282 spin_unlock_irqrestore(&tx_ctx->lock, flags); 283 return ret; 284 } 285 286 static void 287 tx_post_resync_params(struct mlx5e_txqsq *sq, 288 struct mlx5e_ktls_offload_context_tx *priv_tx, 289 u64 rcd_sn) 290 { 291 struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info; 292 __be64 rn_be = cpu_to_be64(rcd_sn); 293 bool skip_static_post; 294 u16 rec_seq_sz; 295 char *rec_seq; 296 297 rec_seq = info->rec_seq; 298 rec_seq_sz = sizeof(info->rec_seq); 299 300 skip_static_post = !memcmp(rec_seq, &rn_be, rec_seq_sz); 301 if (!skip_static_post) 302 memcpy(rec_seq, &rn_be, rec_seq_sz); 303 304 mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, skip_static_post, true); 305 } 306 307 static int 308 tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool first) 309 { 310 struct mlx5_wqe_ctrl_seg *cseg; 311 struct mlx5_wqe_data_seg *dseg; 312 struct mlx5e_dump_wqe *wqe; 313 dma_addr_t dma_addr = 0; 314 u16 ds_cnt; 315 int fsz; 316 u16 pi; 317 318 BUILD_BUG_ON(MLX5E_KTLS_DUMP_WQEBBS != 1); 319 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 320 wqe = MLX5E_TLS_FETCH_DUMP_WQE(sq, pi); 321 322 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; 323 324 cseg = &wqe->ctrl; 325 dseg = &wqe->data; 326 327 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_DUMP); 328 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 329 cseg->tis_tir_num = cpu_to_be32(tisn << 8); 330 cseg->fm_ce_se = first ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; 331 332 fsz = skb_frag_size(frag); 333 dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, 334 DMA_TO_DEVICE); 335 if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) 336 return -ENOMEM; 337 338 dseg->addr = cpu_to_be64(dma_addr); 339 dseg->lkey = sq->mkey_be; 340 dseg->byte_count = cpu_to_be32(fsz); 341 mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE); 342 343 tx_fill_wi(sq, pi, MLX5E_KTLS_DUMP_WQEBBS, fsz, skb_frag_page(frag)); 344 sq->pc += MLX5E_KTLS_DUMP_WQEBBS; 345 346 return 0; 347 } 348 349 void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, 350 struct mlx5e_tx_wqe_info *wi, 351 u32 *dma_fifo_cc) 352 { 353 struct mlx5e_sq_stats *stats; 354 struct mlx5e_sq_dma *dma; 355 356 dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++); 357 stats = sq->stats; 358 359 mlx5e_tx_dma_unmap(sq->pdev, dma); 360 put_page(wi->resync_dump_frag_page); 361 stats->tls_dump_packets++; 362 stats->tls_dump_bytes += wi->num_bytes; 363 } 364 365 static void tx_post_fence_nop(struct mlx5e_txqsq *sq) 366 { 367 struct mlx5_wq_cyc *wq = &sq->wq; 368 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 369 370 tx_fill_wi(sq, pi, 1, 0, NULL); 371 372 mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc); 373 } 374 375 static enum mlx5e_ktls_sync_retval 376 mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx, 377 struct mlx5e_txqsq *sq, 378 int datalen, 379 u32 seq) 380 { 381 struct mlx5e_sq_stats *stats = sq->stats; 382 enum mlx5e_ktls_sync_retval ret; 383 struct tx_sync_info info = {}; 384 int i = 0; 385 386 ret = tx_sync_info_get(priv_tx, seq, datalen, &info); 387 if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) { 388 if (ret == MLX5E_KTLS_SYNC_SKIP_NO_DATA) { 389 stats->tls_skip_no_sync_data++; 390 return MLX5E_KTLS_SYNC_SKIP_NO_DATA; 391 } 392 /* We might get here if a retransmission reaches the driver 393 * after the relevant record is acked. 394 * It should be safe to drop the packet in this case 395 */ 396 stats->tls_drop_no_sync_data++; 397 goto err_out; 398 } 399 400 stats->tls_ooo++; 401 402 tx_post_resync_params(sq, priv_tx, info.rcd_sn); 403 404 /* If no dump WQE was sent, we need to have a fence NOP WQE before the 405 * actual data xmit. 406 */ 407 if (!info.nr_frags) { 408 tx_post_fence_nop(sq); 409 return MLX5E_KTLS_SYNC_DONE; 410 } 411 412 for (; i < info.nr_frags; i++) { 413 unsigned int orig_fsz, frag_offset = 0, n = 0; 414 skb_frag_t *f = &info.frags[i]; 415 416 orig_fsz = skb_frag_size(f); 417 418 do { 419 bool fence = !(i || frag_offset); 420 unsigned int fsz; 421 422 n++; 423 fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset); 424 skb_frag_size_set(f, fsz); 425 if (tx_post_resync_dump(sq, f, priv_tx->tisn, fence)) { 426 page_ref_add(skb_frag_page(f), n - 1); 427 goto err_out; 428 } 429 430 skb_frag_off_add(f, fsz); 431 frag_offset += fsz; 432 } while (frag_offset < orig_fsz); 433 434 page_ref_add(skb_frag_page(f), n - 1); 435 } 436 437 return MLX5E_KTLS_SYNC_DONE; 438 439 err_out: 440 for (; i < info.nr_frags; i++) 441 /* The put_page() here undoes the page ref obtained in tx_sync_info_get(). 442 * Page refs obtained for the DUMP WQEs above (by page_ref_add) will be 443 * released only upon their completions (or in mlx5e_free_txqsq_descs, 444 * if channel closes). 445 */ 446 put_page(skb_frag_page(&info.frags[i])); 447 448 return MLX5E_KTLS_SYNC_FAIL; 449 } 450 451 bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, 452 struct sk_buff *skb, 453 struct mlx5e_accel_tx_tls_state *state) 454 { 455 struct mlx5e_ktls_offload_context_tx *priv_tx; 456 struct mlx5e_sq_stats *stats = sq->stats; 457 struct tls_context *tls_ctx; 458 int datalen; 459 u32 seq; 460 461 datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); 462 if (!datalen) 463 return true; 464 465 mlx5e_tx_mpwqe_ensure_complete(sq); 466 467 tls_ctx = tls_get_ctx(skb->sk); 468 if (WARN_ON_ONCE(tls_ctx->netdev != netdev)) 469 goto err_out; 470 471 priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); 472 473 if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) { 474 mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false); 475 } 476 477 seq = ntohl(tcp_hdr(skb)->seq); 478 if (unlikely(priv_tx->expected_seq != seq)) { 479 enum mlx5e_ktls_sync_retval ret = 480 mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq); 481 482 switch (ret) { 483 case MLX5E_KTLS_SYNC_DONE: 484 break; 485 case MLX5E_KTLS_SYNC_SKIP_NO_DATA: 486 if (likely(!skb->decrypted)) 487 goto out; 488 WARN_ON_ONCE(1); 489 fallthrough; 490 case MLX5E_KTLS_SYNC_FAIL: 491 goto err_out; 492 } 493 } 494 495 priv_tx->expected_seq = seq + datalen; 496 497 state->tls_tisn = priv_tx->tisn; 498 499 stats->tls_encrypted_packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; 500 stats->tls_encrypted_bytes += datalen; 501 502 out: 503 return true; 504 505 err_out: 506 dev_kfree_skb_any(skb); 507 return false; 508 } 509