1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/ipv6.h> 6 #include <linux/skbuff.h> 7 #include <linux/string.h> 8 #include <net/inet6_hashtables.h> 9 #include <net/tls.h> 10 11 #include "../ccm.h" 12 #include "../nfp_net.h" 13 #include "crypto.h" 14 #include "fw.h" 15 16 #define NFP_NET_TLS_CCM_MBOX_OPS_MASK \ 17 (BIT(NFP_CCM_TYPE_CRYPTO_RESET) | \ 18 BIT(NFP_CCM_TYPE_CRYPTO_ADD) | \ 19 BIT(NFP_CCM_TYPE_CRYPTO_DEL) | \ 20 BIT(NFP_CCM_TYPE_CRYPTO_UPDATE)) 21 22 #define NFP_NET_TLS_OPCODE_MASK_RX \ 23 BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC) 24 25 #define NFP_NET_TLS_OPCODE_MASK_TX \ 26 BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC) 27 28 #define NFP_NET_TLS_OPCODE_MASK \ 29 (NFP_NET_TLS_OPCODE_MASK_RX | NFP_NET_TLS_OPCODE_MASK_TX) 30 31 static void nfp_net_crypto_set_op(struct nfp_net *nn, u8 opcode, bool on) 32 { 33 u32 off, val; 34 35 off = nn->tlv_caps.crypto_enable_off + round_down(opcode / 8, 4); 36 37 val = nn_readl(nn, off); 38 if (on) 39 val |= BIT(opcode & 31); 40 else 41 val &= ~BIT(opcode & 31); 42 nn_writel(nn, off, val); 43 } 44 45 static bool 46 __nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add, 47 enum tls_offload_ctx_dir direction) 48 { 49 u8 opcode; 50 int cnt; 51 52 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 53 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 54 nn->ktls_tx_conn_cnt += add; 55 cnt = nn->ktls_tx_conn_cnt; 56 nn->dp.ktls_tx = !!nn->ktls_tx_conn_cnt; 57 } else { 58 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 59 nn->ktls_rx_conn_cnt += add; 60 cnt = nn->ktls_rx_conn_cnt; 61 } 62 63 /* Care only about 0 -> 1 and 1 -> 0 transitions */ 64 if (cnt > 1) 65 return false; 66 67 nfp_net_crypto_set_op(nn, opcode, cnt); 68 return true; 69 } 70 71 static int 72 nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add, 73 enum tls_offload_ctx_dir direction) 74 { 75 int ret = 0; 76 77 /* Use the BAR lock to protect the connection counts */ 78 nn_ctrl_bar_lock(nn); 79 if (__nfp_net_tls_conn_cnt_changed(nn, add, direction)) { 80 ret = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO); 81 /* Undo the cnt adjustment if failed */ 82 if (ret) 83 __nfp_net_tls_conn_cnt_changed(nn, -add, direction); 84 } 85 nn_ctrl_bar_unlock(nn); 86 87 return ret; 88 } 89 90 static int 91 nfp_net_tls_conn_add(struct nfp_net *nn, enum tls_offload_ctx_dir direction) 92 { 93 return nfp_net_tls_conn_cnt_changed(nn, 1, direction); 94 } 95 96 static int 97 nfp_net_tls_conn_remove(struct nfp_net *nn, enum tls_offload_ctx_dir direction) 98 { 99 return nfp_net_tls_conn_cnt_changed(nn, -1, direction); 100 } 101 102 static struct sk_buff * 103 nfp_net_tls_alloc_simple(struct nfp_net *nn, size_t req_sz, gfp_t flags) 104 { 105 return nfp_ccm_mbox_msg_alloc(nn, req_sz, 106 sizeof(struct nfp_crypto_reply_simple), 107 flags); 108 } 109 110 static int 111 nfp_net_tls_communicate_simple(struct nfp_net *nn, struct sk_buff *skb, 112 const char *name, enum nfp_ccm_type type) 113 { 114 struct nfp_crypto_reply_simple *reply; 115 int err; 116 117 err = __nfp_ccm_mbox_communicate(nn, skb, type, 118 sizeof(*reply), sizeof(*reply), 119 type == NFP_CCM_TYPE_CRYPTO_DEL); 120 if (err) { 121 nn_dp_warn(&nn->dp, "failed to %s TLS: %d\n", name, err); 122 return err; 123 } 124 125 reply = (void *)skb->data; 126 err = -be32_to_cpu(reply->error); 127 if (err) 128 nn_dp_warn(&nn->dp, "failed to %s TLS, fw replied: %d\n", 129 name, err); 130 dev_consume_skb_any(skb); 131 132 return err; 133 } 134 135 static void nfp_net_tls_del_fw(struct nfp_net *nn, __be32 *fw_handle) 136 { 137 struct nfp_crypto_req_del *req; 138 struct sk_buff *skb; 139 140 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL); 141 if (!skb) 142 return; 143 144 req = (void *)skb->data; 145 req->ep_id = 0; 146 memcpy(req->handle, fw_handle, sizeof(req->handle)); 147 148 nfp_net_tls_communicate_simple(nn, skb, "delete", 149 NFP_CCM_TYPE_CRYPTO_DEL); 150 } 151 152 static void 153 nfp_net_tls_set_ipver_vlan(struct nfp_crypto_req_add_front *front, u8 ipver) 154 { 155 front->ipver_vlan = cpu_to_be16(FIELD_PREP(NFP_NET_TLS_IPVER, ipver) | 156 FIELD_PREP(NFP_NET_TLS_VLAN, 157 NFP_NET_TLS_VLAN_UNUSED)); 158 } 159 160 static void 161 nfp_net_tls_assign_conn_id(struct nfp_net *nn, 162 struct nfp_crypto_req_add_front *front) 163 { 164 u32 len; 165 u64 id; 166 167 id = atomic64_inc_return(&nn->ktls_conn_id_gen); 168 len = front->key_len - NFP_NET_TLS_NON_ADDR_KEY_LEN; 169 170 memcpy(front->l3_addrs, &id, sizeof(id)); 171 memset(front->l3_addrs + sizeof(id), 0, len - sizeof(id)); 172 } 173 174 static struct nfp_crypto_req_add_back * 175 nfp_net_tls_set_ipv4(struct nfp_net *nn, struct nfp_crypto_req_add_v4 *req, 176 struct sock *sk, int direction) 177 { 178 struct inet_sock *inet = inet_sk(sk); 179 180 req->front.key_len += sizeof(__be32) * 2; 181 182 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 183 nfp_net_tls_assign_conn_id(nn, &req->front); 184 } else { 185 req->src_ip = inet->inet_daddr; 186 req->dst_ip = inet->inet_saddr; 187 } 188 189 return &req->back; 190 } 191 192 static struct nfp_crypto_req_add_back * 193 nfp_net_tls_set_ipv6(struct nfp_net *nn, struct nfp_crypto_req_add_v6 *req, 194 struct sock *sk, int direction) 195 { 196 #if IS_ENABLED(CONFIG_IPV6) 197 struct ipv6_pinfo *np = inet6_sk(sk); 198 199 req->front.key_len += sizeof(struct in6_addr) * 2; 200 201 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 202 nfp_net_tls_assign_conn_id(nn, &req->front); 203 } else { 204 memcpy(req->src_ip, &sk->sk_v6_daddr, sizeof(req->src_ip)); 205 memcpy(req->dst_ip, &np->saddr, sizeof(req->dst_ip)); 206 } 207 208 #endif 209 return &req->back; 210 } 211 212 static void 213 nfp_net_tls_set_l4(struct nfp_crypto_req_add_front *front, 214 struct nfp_crypto_req_add_back *back, struct sock *sk, 215 int direction) 216 { 217 struct inet_sock *inet = inet_sk(sk); 218 219 front->l4_proto = IPPROTO_TCP; 220 221 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 222 back->src_port = 0; 223 back->dst_port = 0; 224 } else { 225 back->src_port = inet->inet_dport; 226 back->dst_port = inet->inet_sport; 227 } 228 } 229 230 static u8 nfp_tls_1_2_dir_to_opcode(enum tls_offload_ctx_dir direction) 231 { 232 switch (direction) { 233 case TLS_OFFLOAD_CTX_DIR_TX: 234 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 235 case TLS_OFFLOAD_CTX_DIR_RX: 236 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 237 default: 238 WARN_ON_ONCE(1); 239 return 0; 240 } 241 } 242 243 static bool 244 nfp_net_cipher_supported(struct nfp_net *nn, u16 cipher_type, 245 enum tls_offload_ctx_dir direction) 246 { 247 u8 bit; 248 249 switch (cipher_type) { 250 case TLS_CIPHER_AES_GCM_128: 251 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 252 bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 253 else 254 bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 255 break; 256 default: 257 return false; 258 } 259 260 return nn->tlv_caps.crypto_ops & BIT(bit); 261 } 262 263 static int 264 nfp_net_tls_add(struct net_device *netdev, struct sock *sk, 265 enum tls_offload_ctx_dir direction, 266 struct tls_crypto_info *crypto_info, 267 u32 start_offload_tcp_sn) 268 { 269 struct tls12_crypto_info_aes_gcm_128 *tls_ci; 270 struct nfp_net *nn = netdev_priv(netdev); 271 struct nfp_crypto_req_add_front *front; 272 struct nfp_net_tls_offload_ctx *ntls; 273 struct nfp_crypto_req_add_back *back; 274 struct nfp_crypto_reply_add *reply; 275 struct sk_buff *skb; 276 size_t req_sz; 277 void *req; 278 bool ipv6; 279 int err; 280 281 BUILD_BUG_ON(sizeof(struct nfp_net_tls_offload_ctx) > 282 TLS_DRIVER_STATE_SIZE_TX); 283 BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) > 284 TLS_DRIVER_STATE_SIZE_RX); 285 286 if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction)) 287 return -EOPNOTSUPP; 288 289 switch (sk->sk_family) { 290 #if IS_ENABLED(CONFIG_IPV6) 291 case AF_INET6: 292 if (ipv6_only_sock(sk) || 293 ipv6_addr_type(&sk->sk_v6_daddr) != IPV6_ADDR_MAPPED) { 294 req_sz = sizeof(struct nfp_crypto_req_add_v6); 295 ipv6 = true; 296 break; 297 } 298 fallthrough; 299 #endif 300 case AF_INET: 301 req_sz = sizeof(struct nfp_crypto_req_add_v4); 302 ipv6 = false; 303 break; 304 default: 305 return -EOPNOTSUPP; 306 } 307 308 err = nfp_net_tls_conn_add(nn, direction); 309 if (err) 310 return err; 311 312 skb = nfp_ccm_mbox_msg_alloc(nn, req_sz, sizeof(*reply), GFP_KERNEL); 313 if (!skb) { 314 err = -ENOMEM; 315 goto err_conn_remove; 316 } 317 318 front = (void *)skb->data; 319 front->ep_id = 0; 320 front->key_len = NFP_NET_TLS_NON_ADDR_KEY_LEN; 321 front->opcode = nfp_tls_1_2_dir_to_opcode(direction); 322 memset(front->resv, 0, sizeof(front->resv)); 323 324 nfp_net_tls_set_ipver_vlan(front, ipv6 ? 6 : 4); 325 326 req = (void *)skb->data; 327 if (ipv6) 328 back = nfp_net_tls_set_ipv6(nn, req, sk, direction); 329 else 330 back = nfp_net_tls_set_ipv4(nn, req, sk, direction); 331 332 nfp_net_tls_set_l4(front, back, sk, direction); 333 334 back->counter = 0; 335 back->tcp_seq = cpu_to_be32(start_offload_tcp_sn); 336 337 tls_ci = (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 338 memcpy(back->key, tls_ci->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE); 339 memset(&back->key[TLS_CIPHER_AES_GCM_128_KEY_SIZE / 4], 0, 340 sizeof(back->key) - TLS_CIPHER_AES_GCM_128_KEY_SIZE); 341 memcpy(back->iv, tls_ci->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE); 342 memcpy(&back->salt, tls_ci->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 343 memcpy(back->rec_no, tls_ci->rec_seq, sizeof(tls_ci->rec_seq)); 344 345 /* Get an extra ref on the skb so we can wipe the key after */ 346 skb_get(skb); 347 348 err = nfp_ccm_mbox_communicate(nn, skb, NFP_CCM_TYPE_CRYPTO_ADD, 349 sizeof(*reply), sizeof(*reply)); 350 reply = (void *)skb->data; 351 352 /* We depend on CCM MBOX code not reallocating skb we sent 353 * so we can clear the key material out of the memory. 354 */ 355 if (!WARN_ON_ONCE((u8 *)back < skb->head || 356 (u8 *)back > skb_end_pointer(skb)) && 357 !WARN_ON_ONCE((u8 *)&reply[1] > (u8 *)back)) 358 memzero_explicit(back, sizeof(*back)); 359 dev_consume_skb_any(skb); /* the extra ref from skb_get() above */ 360 361 if (err) { 362 nn_dp_warn(&nn->dp, "failed to add TLS: %d (%d)\n", 363 err, direction == TLS_OFFLOAD_CTX_DIR_TX); 364 /* communicate frees skb on error */ 365 goto err_conn_remove; 366 } 367 368 err = -be32_to_cpu(reply->error); 369 if (err) { 370 if (err == -ENOSPC) { 371 if (!atomic_fetch_inc(&nn->ktls_no_space)) 372 nn_info(nn, "HW TLS table full\n"); 373 } else { 374 nn_dp_warn(&nn->dp, 375 "failed to add TLS, FW replied: %d\n", err); 376 } 377 goto err_free_skb; 378 } 379 380 if (!reply->handle[0] && !reply->handle[1]) { 381 nn_dp_warn(&nn->dp, "FW returned NULL handle\n"); 382 err = -EINVAL; 383 goto err_fw_remove; 384 } 385 386 ntls = tls_driver_ctx(sk, direction); 387 memcpy(ntls->fw_handle, reply->handle, sizeof(ntls->fw_handle)); 388 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 389 ntls->next_seq = start_offload_tcp_sn; 390 dev_consume_skb_any(skb); 391 392 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 393 return 0; 394 395 if (!nn->tlv_caps.tls_resync_ss) 396 tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT); 397 398 return 0; 399 400 err_fw_remove: 401 nfp_net_tls_del_fw(nn, reply->handle); 402 err_free_skb: 403 dev_consume_skb_any(skb); 404 err_conn_remove: 405 nfp_net_tls_conn_remove(nn, direction); 406 return err; 407 } 408 409 static void 410 nfp_net_tls_del(struct net_device *netdev, struct tls_context *tls_ctx, 411 enum tls_offload_ctx_dir direction) 412 { 413 struct nfp_net *nn = netdev_priv(netdev); 414 struct nfp_net_tls_offload_ctx *ntls; 415 416 nfp_net_tls_conn_remove(nn, direction); 417 418 ntls = __tls_driver_ctx(tls_ctx, direction); 419 nfp_net_tls_del_fw(nn, ntls->fw_handle); 420 } 421 422 static int 423 nfp_net_tls_resync(struct net_device *netdev, struct sock *sk, u32 seq, 424 u8 *rcd_sn, enum tls_offload_ctx_dir direction) 425 { 426 struct nfp_net *nn = netdev_priv(netdev); 427 struct nfp_net_tls_offload_ctx *ntls; 428 struct nfp_crypto_req_update *req; 429 enum nfp_ccm_type type; 430 struct sk_buff *skb; 431 gfp_t flags; 432 int err; 433 434 flags = direction == TLS_OFFLOAD_CTX_DIR_TX ? GFP_KERNEL : GFP_ATOMIC; 435 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), flags); 436 if (!skb) 437 return -ENOMEM; 438 439 ntls = tls_driver_ctx(sk, direction); 440 req = (void *)skb->data; 441 req->ep_id = 0; 442 req->opcode = nfp_tls_1_2_dir_to_opcode(direction); 443 memset(req->resv, 0, sizeof(req->resv)); 444 memcpy(req->handle, ntls->fw_handle, sizeof(ntls->fw_handle)); 445 req->tcp_seq = cpu_to_be32(seq); 446 memcpy(req->rec_no, rcd_sn, sizeof(req->rec_no)); 447 448 type = NFP_CCM_TYPE_CRYPTO_UPDATE; 449 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 450 err = nfp_net_tls_communicate_simple(nn, skb, "sync", type); 451 if (err) 452 return err; 453 ntls->next_seq = seq; 454 } else { 455 if (nn->tlv_caps.tls_resync_ss) 456 type = NFP_CCM_TYPE_CRYPTO_RESYNC; 457 nfp_ccm_mbox_post(nn, skb, type, 458 sizeof(struct nfp_crypto_reply_simple)); 459 atomic_inc(&nn->ktls_rx_resync_sent); 460 } 461 462 return 0; 463 } 464 465 static const struct tlsdev_ops nfp_net_tls_ops = { 466 .tls_dev_add = nfp_net_tls_add, 467 .tls_dev_del = nfp_net_tls_del, 468 .tls_dev_resync = nfp_net_tls_resync, 469 }; 470 471 int nfp_net_tls_rx_resync_req(struct net_device *netdev, 472 struct nfp_net_tls_resync_req *req, 473 void *pkt, unsigned int pkt_len) 474 { 475 struct nfp_net *nn = netdev_priv(netdev); 476 struct nfp_net_tls_offload_ctx *ntls; 477 struct ipv6hdr *ipv6h; 478 struct tcphdr *th; 479 struct iphdr *iph; 480 struct sock *sk; 481 __be32 tcp_seq; 482 int err; 483 484 iph = pkt + req->l3_offset; 485 ipv6h = pkt + req->l3_offset; 486 th = pkt + req->l4_offset; 487 488 if ((u8 *)&th[1] > (u8 *)pkt + pkt_len) { 489 netdev_warn_once(netdev, "invalid TLS RX resync request (l3_off: %hhu l4_off: %hhu pkt_len: %u)\n", 490 req->l3_offset, req->l4_offset, pkt_len); 491 err = -EINVAL; 492 goto err_cnt_ign; 493 } 494 495 switch (ipv6h->version) { 496 case 4: 497 sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo, 498 iph->saddr, th->source, iph->daddr, 499 th->dest, netdev->ifindex); 500 break; 501 #if IS_ENABLED(CONFIG_IPV6) 502 case 6: 503 sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo, 504 &ipv6h->saddr, th->source, 505 &ipv6h->daddr, ntohs(th->dest), 506 netdev->ifindex, 0); 507 break; 508 #endif 509 default: 510 netdev_warn_once(netdev, "invalid TLS RX resync request (l3_off: %hhu l4_off: %hhu ipver: %u)\n", 511 req->l3_offset, req->l4_offset, iph->version); 512 err = -EINVAL; 513 goto err_cnt_ign; 514 } 515 516 err = 0; 517 if (!sk) 518 goto err_cnt_ign; 519 if (!tls_is_sk_rx_device_offloaded(sk) || 520 sk->sk_shutdown & RCV_SHUTDOWN) 521 goto err_put_sock; 522 523 ntls = tls_driver_ctx(sk, TLS_OFFLOAD_CTX_DIR_RX); 524 /* some FW versions can't report the handle and report 0s */ 525 if (memchr_inv(&req->fw_handle, 0, sizeof(req->fw_handle)) && 526 memcmp(&req->fw_handle, &ntls->fw_handle, sizeof(ntls->fw_handle))) 527 goto err_put_sock; 528 529 /* copy to ensure alignment */ 530 memcpy(&tcp_seq, &req->tcp_seq, sizeof(tcp_seq)); 531 tls_offload_rx_resync_request(sk, tcp_seq); 532 atomic_inc(&nn->ktls_rx_resync_req); 533 534 sock_gen_put(sk); 535 return 0; 536 537 err_put_sock: 538 sock_gen_put(sk); 539 err_cnt_ign: 540 atomic_inc(&nn->ktls_rx_resync_ign); 541 return err; 542 } 543 544 static int nfp_net_tls_reset(struct nfp_net *nn) 545 { 546 struct nfp_crypto_req_reset *req; 547 struct sk_buff *skb; 548 549 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL); 550 if (!skb) 551 return -ENOMEM; 552 553 req = (void *)skb->data; 554 req->ep_id = 0; 555 556 return nfp_net_tls_communicate_simple(nn, skb, "reset", 557 NFP_CCM_TYPE_CRYPTO_RESET); 558 } 559 560 int nfp_net_tls_init(struct nfp_net *nn) 561 { 562 struct net_device *netdev = nn->dp.netdev; 563 int err; 564 565 if (!(nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK)) 566 return 0; 567 568 if ((nn->tlv_caps.mbox_cmsg_types & NFP_NET_TLS_CCM_MBOX_OPS_MASK) != 569 NFP_NET_TLS_CCM_MBOX_OPS_MASK) 570 return 0; 571 572 if (!nfp_ccm_mbox_fits(nn, sizeof(struct nfp_crypto_req_add_v6))) { 573 nn_warn(nn, "disabling TLS offload - mbox too small: %d\n", 574 nn->tlv_caps.mbox_len); 575 return 0; 576 } 577 578 err = nfp_net_tls_reset(nn); 579 if (err) 580 return err; 581 582 nn_ctrl_bar_lock(nn); 583 nn_writel(nn, nn->tlv_caps.crypto_enable_off, 0); 584 err = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO); 585 nn_ctrl_bar_unlock(nn); 586 if (err) 587 return err; 588 589 if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_RX) { 590 netdev->hw_features |= NETIF_F_HW_TLS_RX; 591 netdev->features |= NETIF_F_HW_TLS_RX; 592 } 593 if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_TX) { 594 netdev->hw_features |= NETIF_F_HW_TLS_TX; 595 netdev->features |= NETIF_F_HW_TLS_TX; 596 } 597 598 netdev->tlsdev_ops = &nfp_net_tls_ops; 599 600 return 0; 601 } 602