1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include <net/inet6_hashtables.h> 5 #include "en_accel/en_accel.h" 6 #include "en_accel/tls.h" 7 #include "en_accel/ktls_txrx.h" 8 #include "en_accel/ktls_utils.h" 9 #include "en_accel/fs_tcp.h" 10 11 struct accel_rule { 12 struct work_struct work; 13 struct mlx5e_priv *priv; 14 struct mlx5_flow_handle *rule; 15 }; 16 17 #define PROGRESS_PARAMS_WRITE_UNIT 64 18 #define PROGRESS_PARAMS_PADDED_SIZE \ 19 (ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \ 20 PROGRESS_PARAMS_WRITE_UNIT)) 21 22 struct mlx5e_ktls_rx_resync_buf { 23 union { 24 struct mlx5_wqe_tls_progress_params_seg progress; 25 u8 pad[PROGRESS_PARAMS_PADDED_SIZE]; 26 } ____cacheline_aligned_in_smp; 27 dma_addr_t dma_addr; 28 struct mlx5e_ktls_offload_context_rx *priv_rx; 29 }; 30 31 enum { 32 MLX5E_PRIV_RX_FLAG_DELETING, 33 MLX5E_NUM_PRIV_RX_FLAGS, 34 }; 35 36 struct mlx5e_ktls_rx_resync_ctx { 37 struct tls_offload_resync_async core; 38 struct work_struct work; 39 struct mlx5e_priv *priv; 40 refcount_t refcnt; 41 __be64 sw_rcd_sn_be; 42 u32 seq; 43 }; 44 45 struct mlx5e_ktls_offload_context_rx { 46 struct tls12_crypto_info_aes_gcm_128 crypto_info; 47 struct accel_rule rule; 48 struct sock *sk; 49 struct mlx5e_rq_stats *rq_stats; 50 struct mlx5e_tls_sw_stats *sw_stats; 51 struct completion add_ctx; 52 struct mlx5e_tir tir; 53 u32 key_id; 54 u32 rxq; 55 DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS); 56 57 /* resync */ 58 struct mlx5e_ktls_rx_resync_ctx resync; 59 struct list_head list; 60 }; 61 62 static bool mlx5e_ktls_priv_rx_put(struct mlx5e_ktls_offload_context_rx *priv_rx) 63 { 64 if (!refcount_dec_and_test(&priv_rx->resync.refcnt)) 65 return false; 66 67 kfree(priv_rx); 68 return true; 69 } 70 71 static void mlx5e_ktls_priv_rx_get(struct mlx5e_ktls_offload_context_rx *priv_rx) 72 { 73 refcount_inc(&priv_rx->resync.refcnt); 74 } 75 76 struct mlx5e_ktls_resync_resp { 77 /* protects list changes */ 78 spinlock_t lock; 79 struct list_head list; 80 }; 81 82 void mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp *resp_list) 83 { 84 kvfree(resp_list); 85 } 86 87 struct mlx5e_ktls_resync_resp * 88 mlx5e_ktls_rx_resync_create_resp_list(void) 89 { 90 struct mlx5e_ktls_resync_resp *resp_list; 91 92 resp_list = kvzalloc(sizeof(*resp_list), GFP_KERNEL); 93 if (!resp_list) 94 return ERR_PTR(-ENOMEM); 95 96 INIT_LIST_HEAD(&resp_list->list); 97 spin_lock_init(&resp_list->lock); 98 99 return resp_list; 100 } 101 102 static int mlx5e_ktls_create_tir(struct mlx5_core_dev *mdev, struct mlx5e_tir *tir, u32 rqtn) 103 { 104 struct mlx5e_tir_builder *builder; 105 int err; 106 107 builder = mlx5e_tir_builder_alloc(false); 108 if (!builder) 109 return -ENOMEM; 110 111 mlx5e_tir_builder_build_rqt(builder, mdev->mlx5e_res.hw_objs.td.tdn, rqtn, false); 112 mlx5e_tir_builder_build_direct(builder); 113 mlx5e_tir_builder_build_tls(builder); 114 err = mlx5e_tir_init(tir, builder, mdev, false); 115 116 mlx5e_tir_builder_free(builder); 117 118 return err; 119 } 120 121 static void accel_rule_handle_work(struct work_struct *work) 122 { 123 struct mlx5e_ktls_offload_context_rx *priv_rx; 124 struct accel_rule *accel_rule; 125 struct mlx5_flow_handle *rule; 126 127 accel_rule = container_of(work, struct accel_rule, work); 128 priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule); 129 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) 130 goto out; 131 132 rule = mlx5e_accel_fs_add_sk(accel_rule->priv, priv_rx->sk, 133 mlx5e_tir_get_tirn(&priv_rx->tir), 134 MLX5_FS_DEFAULT_FLOW_TAG); 135 if (!IS_ERR_OR_NULL(rule)) 136 accel_rule->rule = rule; 137 out: 138 complete(&priv_rx->add_ctx); 139 } 140 141 static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv) 142 { 143 INIT_WORK(&rule->work, accel_rule_handle_work); 144 rule->priv = priv; 145 } 146 147 static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi, 148 struct mlx5e_icosq_wqe_info *wi) 149 { 150 sq->db.wqe_info[pi] = *wi; 151 } 152 153 static struct mlx5_wqe_ctrl_seg * 154 post_static_params(struct mlx5e_icosq *sq, 155 struct mlx5e_ktls_offload_context_rx *priv_rx) 156 { 157 struct mlx5e_set_tls_static_params_wqe *wqe; 158 struct mlx5e_icosq_wqe_info wi; 159 u16 pi, num_wqebbs; 160 161 num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS; 162 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, num_wqebbs))) 163 return ERR_PTR(-ENOSPC); 164 165 pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); 166 wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); 167 mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info, 168 mlx5e_tir_get_tirn(&priv_rx->tir), 169 priv_rx->key_id, priv_rx->resync.seq, false, 170 TLS_OFFLOAD_CTX_DIR_RX); 171 wi = (struct mlx5e_icosq_wqe_info) { 172 .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS, 173 .num_wqebbs = num_wqebbs, 174 .tls_set_params.priv_rx = priv_rx, 175 }; 176 icosq_fill_wi(sq, pi, &wi); 177 sq->pc += num_wqebbs; 178 179 return &wqe->ctrl; 180 } 181 182 static struct mlx5_wqe_ctrl_seg * 183 post_progress_params(struct mlx5e_icosq *sq, 184 struct mlx5e_ktls_offload_context_rx *priv_rx, 185 u32 next_record_tcp_sn) 186 { 187 struct mlx5e_set_tls_progress_params_wqe *wqe; 188 struct mlx5e_icosq_wqe_info wi; 189 u16 pi, num_wqebbs; 190 191 num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; 192 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, num_wqebbs))) 193 return ERR_PTR(-ENOSPC); 194 195 pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); 196 wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); 197 mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, 198 mlx5e_tir_get_tirn(&priv_rx->tir), 199 false, next_record_tcp_sn, 200 TLS_OFFLOAD_CTX_DIR_RX); 201 wi = (struct mlx5e_icosq_wqe_info) { 202 .wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS, 203 .num_wqebbs = num_wqebbs, 204 .tls_set_params.priv_rx = priv_rx, 205 }; 206 207 icosq_fill_wi(sq, pi, &wi); 208 sq->pc += num_wqebbs; 209 210 return &wqe->ctrl; 211 } 212 213 static int post_rx_param_wqes(struct mlx5e_channel *c, 214 struct mlx5e_ktls_offload_context_rx *priv_rx, 215 u32 next_record_tcp_sn) 216 { 217 struct mlx5_wqe_ctrl_seg *cseg; 218 struct mlx5e_icosq *sq; 219 int err; 220 221 err = 0; 222 sq = &c->async_icosq; 223 spin_lock_bh(&c->async_icosq_lock); 224 225 cseg = post_static_params(sq, priv_rx); 226 if (IS_ERR(cseg)) 227 goto err_out; 228 cseg = post_progress_params(sq, priv_rx, next_record_tcp_sn); 229 if (IS_ERR(cseg)) 230 goto err_out; 231 232 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 233 unlock: 234 spin_unlock_bh(&c->async_icosq_lock); 235 236 return err; 237 238 err_out: 239 priv_rx->rq_stats->tls_resync_req_skip++; 240 err = PTR_ERR(cseg); 241 complete(&priv_rx->add_ctx); 242 goto unlock; 243 } 244 245 static void 246 mlx5e_set_ktls_rx_priv_ctx(struct tls_context *tls_ctx, 247 struct mlx5e_ktls_offload_context_rx *priv_rx) 248 { 249 struct mlx5e_ktls_offload_context_rx **ctx = 250 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX); 251 252 BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_rx *) > 253 TLS_OFFLOAD_CONTEXT_SIZE_RX); 254 255 *ctx = priv_rx; 256 } 257 258 static struct mlx5e_ktls_offload_context_rx * 259 mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx) 260 { 261 struct mlx5e_ktls_offload_context_rx **ctx = 262 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX); 263 264 return *ctx; 265 } 266 267 /* Re-sync */ 268 /* Runs in work context */ 269 static int 270 resync_post_get_progress_params(struct mlx5e_icosq *sq, 271 struct mlx5e_ktls_offload_context_rx *priv_rx) 272 { 273 struct mlx5e_get_tls_progress_params_wqe *wqe; 274 struct mlx5e_ktls_rx_resync_buf *buf; 275 struct mlx5e_icosq_wqe_info wi; 276 struct mlx5_wqe_ctrl_seg *cseg; 277 struct mlx5_seg_get_psv *psv; 278 struct device *pdev; 279 int err; 280 u16 pi; 281 282 buf = kzalloc(sizeof(*buf), GFP_KERNEL); 283 if (unlikely(!buf)) { 284 err = -ENOMEM; 285 goto err_out; 286 } 287 288 pdev = mlx5_core_dma_dev(sq->channel->priv->mdev); 289 buf->dma_addr = dma_map_single(pdev, &buf->progress, 290 PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); 291 if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) { 292 err = -ENOMEM; 293 goto err_free; 294 } 295 296 buf->priv_rx = priv_rx; 297 298 spin_lock_bh(&sq->channel->async_icosq_lock); 299 300 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS))) { 301 spin_unlock_bh(&sq->channel->async_icosq_lock); 302 err = -ENOSPC; 303 goto err_dma_unmap; 304 } 305 306 pi = mlx5e_icosq_get_next_pi(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS); 307 wqe = MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi); 308 309 #define GET_PSV_DS_CNT (DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS)) 310 311 cseg = &wqe->ctrl; 312 cseg->opmod_idx_opcode = 313 cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_GET_PSV | 314 (MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS << 24)); 315 cseg->qpn_ds = 316 cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | GET_PSV_DS_CNT); 317 318 psv = &wqe->psv; 319 psv->num_psv = 1 << 4; 320 psv->l_key = sq->channel->mkey_be; 321 psv->psv_index[0] = cpu_to_be32(mlx5e_tir_get_tirn(&priv_rx->tir)); 322 psv->va = cpu_to_be64(buf->dma_addr); 323 324 wi = (struct mlx5e_icosq_wqe_info) { 325 .wqe_type = MLX5E_ICOSQ_WQE_GET_PSV_TLS, 326 .num_wqebbs = MLX5E_KTLS_GET_PROGRESS_WQEBBS, 327 .tls_get_params.buf = buf, 328 }; 329 icosq_fill_wi(sq, pi, &wi); 330 sq->pc++; 331 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 332 spin_unlock_bh(&sq->channel->async_icosq_lock); 333 334 return 0; 335 336 err_dma_unmap: 337 dma_unmap_single(pdev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); 338 err_free: 339 kfree(buf); 340 err_out: 341 priv_rx->rq_stats->tls_resync_req_skip++; 342 return err; 343 } 344 345 /* Function is called with elevated refcount. 346 * It decreases it only if no WQE is posted. 347 */ 348 static void resync_handle_work(struct work_struct *work) 349 { 350 struct mlx5e_ktls_offload_context_rx *priv_rx; 351 struct mlx5e_ktls_rx_resync_ctx *resync; 352 struct mlx5e_channel *c; 353 struct mlx5e_icosq *sq; 354 355 resync = container_of(work, struct mlx5e_ktls_rx_resync_ctx, work); 356 priv_rx = container_of(resync, struct mlx5e_ktls_offload_context_rx, resync); 357 358 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { 359 mlx5e_ktls_priv_rx_put(priv_rx); 360 return; 361 } 362 363 c = resync->priv->channels.c[priv_rx->rxq]; 364 sq = &c->async_icosq; 365 366 if (resync_post_get_progress_params(sq, priv_rx)) 367 mlx5e_ktls_priv_rx_put(priv_rx); 368 } 369 370 static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync, 371 struct mlx5e_priv *priv) 372 { 373 INIT_WORK(&resync->work, resync_handle_work); 374 resync->priv = priv; 375 refcount_set(&resync->refcnt, 1); 376 } 377 378 /* Function can be called with the refcount being either elevated or not. 379 * It does not affect the refcount. 380 */ 381 static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx, 382 struct mlx5e_channel *c) 383 { 384 struct tls12_crypto_info_aes_gcm_128 *info = &priv_rx->crypto_info; 385 struct mlx5e_ktls_resync_resp *ktls_resync; 386 struct mlx5e_icosq *sq; 387 bool trigger_poll; 388 389 memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq)); 390 391 sq = &c->async_icosq; 392 ktls_resync = sq->ktls_resync; 393 394 spin_lock_bh(&ktls_resync->lock); 395 list_add_tail(&priv_rx->list, &ktls_resync->list); 396 trigger_poll = !test_and_set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state); 397 spin_unlock_bh(&ktls_resync->lock); 398 399 if (!trigger_poll) 400 return; 401 402 if (!napi_if_scheduled_mark_missed(&c->napi)) { 403 spin_lock_bh(&c->async_icosq_lock); 404 mlx5e_trigger_irq(sq); 405 spin_unlock_bh(&c->async_icosq_lock); 406 } 407 } 408 409 /* Function can be called with the refcount being either elevated or not. 410 * It decreases the refcount and may free the kTLS priv context. 411 * Refcount is not elevated only if tls_dev_del has been called, but GET_PSV was 412 * already in flight. 413 */ 414 void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, 415 struct mlx5e_icosq *sq) 416 { 417 struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf; 418 struct mlx5e_ktls_offload_context_rx *priv_rx; 419 struct mlx5e_ktls_rx_resync_ctx *resync; 420 u8 tracker_state, auth_state, *ctx; 421 struct device *dev; 422 u32 hw_seq; 423 424 priv_rx = buf->priv_rx; 425 resync = &priv_rx->resync; 426 dev = mlx5_core_dma_dev(resync->priv->mdev); 427 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) 428 goto out; 429 430 dma_sync_single_for_cpu(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, 431 DMA_FROM_DEVICE); 432 433 ctx = buf->progress.ctx; 434 tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state); 435 auth_state = MLX5_GET(tls_progress_params, ctx, auth_state); 436 if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING || 437 auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) { 438 priv_rx->rq_stats->tls_resync_req_skip++; 439 goto out; 440 } 441 442 hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn); 443 tls_offload_rx_resync_async_request_end(priv_rx->sk, cpu_to_be32(hw_seq)); 444 priv_rx->rq_stats->tls_resync_req_end++; 445 out: 446 mlx5e_ktls_priv_rx_put(priv_rx); 447 dma_unmap_single(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); 448 kfree(buf); 449 } 450 451 /* Runs in NAPI. 452 * Function elevates the refcount, unless no work is queued. 453 */ 454 static bool resync_queue_get_psv(struct sock *sk) 455 { 456 struct mlx5e_ktls_offload_context_rx *priv_rx; 457 struct mlx5e_ktls_rx_resync_ctx *resync; 458 459 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk)); 460 if (unlikely(!priv_rx)) 461 return false; 462 463 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) 464 return false; 465 466 resync = &priv_rx->resync; 467 mlx5e_ktls_priv_rx_get(priv_rx); 468 if (unlikely(!queue_work(resync->priv->tls->rx_wq, &resync->work))) 469 mlx5e_ktls_priv_rx_put(priv_rx); 470 471 return true; 472 } 473 474 /* Runs in NAPI */ 475 static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) 476 { 477 struct ethhdr *eth = (struct ethhdr *)(skb->data); 478 struct net_device *netdev = rq->netdev; 479 struct sock *sk = NULL; 480 unsigned int datalen; 481 struct iphdr *iph; 482 struct tcphdr *th; 483 __be32 seq; 484 int depth = 0; 485 486 __vlan_get_protocol(skb, eth->h_proto, &depth); 487 iph = (struct iphdr *)(skb->data + depth); 488 489 if (iph->version == 4) { 490 depth += sizeof(struct iphdr); 491 th = (void *)iph + sizeof(struct iphdr); 492 493 sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo, 494 iph->saddr, th->source, iph->daddr, 495 th->dest, netdev->ifindex); 496 #if IS_ENABLED(CONFIG_IPV6) 497 } else { 498 struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph; 499 500 depth += sizeof(struct ipv6hdr); 501 th = (void *)ipv6h + sizeof(struct ipv6hdr); 502 503 sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo, 504 &ipv6h->saddr, th->source, 505 &ipv6h->daddr, ntohs(th->dest), 506 netdev->ifindex, 0); 507 #endif 508 } 509 510 depth += sizeof(struct tcphdr); 511 512 if (unlikely(!sk)) 513 return; 514 515 if (unlikely(sk->sk_state == TCP_TIME_WAIT)) 516 goto unref; 517 518 if (unlikely(!resync_queue_get_psv(sk))) 519 goto unref; 520 521 seq = th->seq; 522 datalen = skb->len - depth; 523 tls_offload_rx_resync_async_request_start(sk, seq, datalen); 524 rq->stats->tls_resync_req_start++; 525 526 unref: 527 sock_gen_put(sk); 528 } 529 530 void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, 531 u32 seq, u8 *rcd_sn) 532 { 533 struct mlx5e_ktls_offload_context_rx *priv_rx; 534 struct mlx5e_ktls_rx_resync_ctx *resync; 535 struct mlx5e_priv *priv; 536 struct mlx5e_channel *c; 537 538 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk)); 539 if (unlikely(!priv_rx)) 540 return; 541 542 resync = &priv_rx->resync; 543 resync->sw_rcd_sn_be = *(__be64 *)rcd_sn; 544 resync->seq = seq; 545 546 priv = netdev_priv(netdev); 547 c = priv->channels.c[priv_rx->rxq]; 548 549 resync_handle_seq_match(priv_rx, c); 550 } 551 552 /* End of resync section */ 553 554 void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, 555 struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) 556 { 557 struct mlx5e_rq_stats *stats = rq->stats; 558 559 switch (get_cqe_tls_offload(cqe)) { 560 case CQE_TLS_OFFLOAD_DECRYPTED: 561 skb->decrypted = 1; 562 stats->tls_decrypted_packets++; 563 stats->tls_decrypted_bytes += *cqe_bcnt; 564 break; 565 case CQE_TLS_OFFLOAD_RESYNC: 566 stats->tls_resync_req_pkt++; 567 resync_update_sn(rq, skb); 568 break; 569 default: /* CQE_TLS_OFFLOAD_ERROR: */ 570 stats->tls_err++; 571 break; 572 } 573 } 574 575 void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi) 576 { 577 struct mlx5e_ktls_offload_context_rx *priv_rx = wi->tls_set_params.priv_rx; 578 struct accel_rule *rule = &priv_rx->rule; 579 580 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { 581 complete(&priv_rx->add_ctx); 582 return; 583 } 584 queue_work(rule->priv->tls->rx_wq, &rule->work); 585 } 586 587 static int mlx5e_ktls_sk_get_rxq(struct sock *sk) 588 { 589 int rxq = sk_rx_queue_get(sk); 590 591 if (unlikely(rxq == -1)) 592 rxq = 0; 593 594 return rxq; 595 } 596 597 int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, 598 struct tls_crypto_info *crypto_info, 599 u32 start_offload_tcp_sn) 600 { 601 struct mlx5e_ktls_offload_context_rx *priv_rx; 602 struct mlx5e_ktls_rx_resync_ctx *resync; 603 struct tls_context *tls_ctx; 604 struct mlx5_core_dev *mdev; 605 struct mlx5e_priv *priv; 606 int rxq, err; 607 u32 rqtn; 608 609 tls_ctx = tls_get_ctx(sk); 610 priv = netdev_priv(netdev); 611 mdev = priv->mdev; 612 priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL); 613 if (unlikely(!priv_rx)) 614 return -ENOMEM; 615 616 err = mlx5_ktls_create_key(mdev, crypto_info, &priv_rx->key_id); 617 if (err) 618 goto err_create_key; 619 620 priv_rx->crypto_info = 621 *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 622 623 rxq = mlx5e_ktls_sk_get_rxq(sk); 624 priv_rx->rxq = rxq; 625 priv_rx->sk = sk; 626 627 priv_rx->rq_stats = &priv->channel_stats[rxq].rq; 628 priv_rx->sw_stats = &priv->tls->sw_stats; 629 mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx); 630 631 rqtn = mlx5e_rx_res_get_rqtn_direct(priv->rx_res, rxq); 632 633 err = mlx5e_ktls_create_tir(mdev, &priv_rx->tir, rqtn); 634 if (err) 635 goto err_create_tir; 636 637 init_completion(&priv_rx->add_ctx); 638 639 accel_rule_init(&priv_rx->rule, priv); 640 resync = &priv_rx->resync; 641 resync_init(resync, priv); 642 tls_offload_ctx_rx(tls_ctx)->resync_async = &resync->core; 643 tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC); 644 645 err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn); 646 if (err) 647 goto err_post_wqes; 648 649 atomic64_inc(&priv_rx->sw_stats->rx_tls_ctx); 650 651 return 0; 652 653 err_post_wqes: 654 mlx5e_tir_destroy(&priv_rx->tir); 655 err_create_tir: 656 mlx5_ktls_destroy_key(mdev, priv_rx->key_id); 657 err_create_key: 658 kfree(priv_rx); 659 return err; 660 } 661 662 void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) 663 { 664 struct mlx5e_ktls_offload_context_rx *priv_rx; 665 struct mlx5e_ktls_rx_resync_ctx *resync; 666 struct mlx5_core_dev *mdev; 667 struct mlx5e_priv *priv; 668 669 priv = netdev_priv(netdev); 670 mdev = priv->mdev; 671 672 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx); 673 set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags); 674 mlx5e_set_ktls_rx_priv_ctx(tls_ctx, NULL); 675 synchronize_net(); /* Sync with NAPI */ 676 if (!cancel_work_sync(&priv_rx->rule.work)) 677 /* completion is needed, as the priv_rx in the add flow 678 * is maintained on the wqe info (wi), not on the socket. 679 */ 680 wait_for_completion(&priv_rx->add_ctx); 681 resync = &priv_rx->resync; 682 if (cancel_work_sync(&resync->work)) 683 mlx5e_ktls_priv_rx_put(priv_rx); 684 685 atomic64_inc(&priv_rx->sw_stats->rx_tls_del); 686 if (priv_rx->rule.rule) 687 mlx5e_accel_fs_del_sk(priv_rx->rule.rule); 688 689 mlx5e_tir_destroy(&priv_rx->tir); 690 mlx5_ktls_destroy_key(mdev, priv_rx->key_id); 691 /* priv_rx should normally be freed here, but if there is an outstanding 692 * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is 693 * processed. 694 */ 695 mlx5e_ktls_priv_rx_put(priv_rx); 696 } 697 698 bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_channel *c, int budget) 699 { 700 struct mlx5e_ktls_offload_context_rx *priv_rx, *tmp; 701 struct mlx5e_ktls_resync_resp *ktls_resync; 702 struct mlx5_wqe_ctrl_seg *db_cseg; 703 struct mlx5e_icosq *sq; 704 LIST_HEAD(local_list); 705 int i, j; 706 707 sq = &c->async_icosq; 708 709 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) 710 return false; 711 712 ktls_resync = sq->ktls_resync; 713 db_cseg = NULL; 714 i = 0; 715 716 spin_lock(&ktls_resync->lock); 717 list_for_each_entry_safe(priv_rx, tmp, &ktls_resync->list, list) { 718 list_move(&priv_rx->list, &local_list); 719 if (++i == budget) 720 break; 721 } 722 if (list_empty(&ktls_resync->list)) 723 clear_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state); 724 spin_unlock(&ktls_resync->lock); 725 726 spin_lock(&c->async_icosq_lock); 727 for (j = 0; j < i; j++) { 728 struct mlx5_wqe_ctrl_seg *cseg; 729 730 priv_rx = list_first_entry(&local_list, 731 struct mlx5e_ktls_offload_context_rx, 732 list); 733 cseg = post_static_params(sq, priv_rx); 734 if (IS_ERR(cseg)) 735 break; 736 list_del(&priv_rx->list); 737 db_cseg = cseg; 738 } 739 if (db_cseg) 740 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, db_cseg); 741 spin_unlock(&c->async_icosq_lock); 742 743 priv_rx->rq_stats->tls_resync_res_ok += j; 744 745 if (!list_empty(&local_list)) { 746 /* This happens only if ICOSQ is full. 747 * There is no need to mark busy or explicitly ask for a NAPI cycle, 748 * it will be triggered by the outstanding ICOSQ completions. 749 */ 750 spin_lock(&ktls_resync->lock); 751 list_splice(&local_list, &ktls_resync->list); 752 set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state); 753 spin_unlock(&ktls_resync->lock); 754 priv_rx->rq_stats->tls_resync_res_retry++; 755 } 756 757 return i == budget; 758 } 759