1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc */ 3 /* Copyright (C) 2021 Corigine, Inc */ 4 5 #include <linux/module.h> 6 #include <linux/kernel.h> 7 #include <linux/init.h> 8 #include <linux/netdevice.h> 9 #include <asm/unaligned.h> 10 #include <linux/ktime.h> 11 #include <net/xfrm.h> 12 13 #include "../nfp_net_ctrl.h" 14 #include "../nfp_net.h" 15 #include "crypto.h" 16 17 #define NFP_NET_IPSEC_MAX_SA_CNT (16 * 1024) /* Firmware support a maximum of 16K SA offload */ 18 19 /* IPsec config message cmd codes */ 20 enum nfp_ipsec_cfg_mssg_cmd_codes { 21 NFP_IPSEC_CFG_MSSG_ADD_SA, /* Add a new SA */ 22 NFP_IPSEC_CFG_MSSG_INV_SA /* Invalidate an existing SA */ 23 }; 24 25 /* IPsec config message response codes */ 26 enum nfp_ipsec_cfg_mssg_rsp_codes { 27 NFP_IPSEC_CFG_MSSG_OK, 28 NFP_IPSEC_CFG_MSSG_FAILED, 29 NFP_IPSEC_CFG_MSSG_SA_VALID, 30 NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED, 31 NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED, 32 NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD 33 }; 34 35 /* Protocol */ 36 enum nfp_ipsec_sa_prot { 37 NFP_IPSEC_PROTOCOL_AH = 0, 38 NFP_IPSEC_PROTOCOL_ESP = 1 39 }; 40 41 /* Mode */ 42 enum nfp_ipsec_sa_mode { 43 NFP_IPSEC_PROTMODE_TRANSPORT = 0, 44 NFP_IPSEC_PROTMODE_TUNNEL = 1 45 }; 46 47 /* Cipher types */ 48 enum nfp_ipsec_sa_cipher { 49 NFP_IPSEC_CIPHER_NULL, 50 NFP_IPSEC_CIPHER_3DES, 51 NFP_IPSEC_CIPHER_AES128, 52 NFP_IPSEC_CIPHER_AES192, 53 NFP_IPSEC_CIPHER_AES256, 54 NFP_IPSEC_CIPHER_AES128_NULL, 55 NFP_IPSEC_CIPHER_AES192_NULL, 56 NFP_IPSEC_CIPHER_AES256_NULL, 57 NFP_IPSEC_CIPHER_CHACHA20 58 }; 59 60 /* Cipher modes */ 61 enum nfp_ipsec_sa_cipher_mode { 62 NFP_IPSEC_CIMODE_ECB, 63 NFP_IPSEC_CIMODE_CBC, 64 NFP_IPSEC_CIMODE_CFB, 65 NFP_IPSEC_CIMODE_OFB, 66 NFP_IPSEC_CIMODE_CTR 67 }; 68 69 /* Hash types */ 70 enum nfp_ipsec_sa_hash_type { 71 NFP_IPSEC_HASH_NONE, 72 NFP_IPSEC_HASH_MD5_96, 73 NFP_IPSEC_HASH_SHA1_96, 74 NFP_IPSEC_HASH_SHA256_96, 75 NFP_IPSEC_HASH_SHA384_96, 76 NFP_IPSEC_HASH_SHA512_96, 77 NFP_IPSEC_HASH_MD5_128, 78 NFP_IPSEC_HASH_SHA1_80, 79 NFP_IPSEC_HASH_SHA256_128, 80 NFP_IPSEC_HASH_SHA384_192, 81 NFP_IPSEC_HASH_SHA512_256, 82 NFP_IPSEC_HASH_GF128_128, 83 NFP_IPSEC_HASH_POLY1305_128 84 }; 85 86 /* IPSEC_CFG_MSSG_ADD_SA */ 87 struct nfp_ipsec_cfg_add_sa { 88 u32 ciph_key[8]; /* Cipher Key */ 89 union { 90 u32 auth_key[16]; /* Authentication Key */ 91 struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */ 92 u32 salt; /* Initialized with SA */ 93 u32 resv[15]; 94 } aesgcm_fields; 95 }; 96 struct sa_ctrl_word { 97 uint32_t hash :4; /* From nfp_ipsec_sa_hash_type */ 98 uint32_t cimode :4; /* From nfp_ipsec_sa_cipher_mode */ 99 uint32_t cipher :4; /* From nfp_ipsec_sa_cipher */ 100 uint32_t mode :2; /* From nfp_ipsec_sa_mode */ 101 uint32_t proto :2; /* From nfp_ipsec_sa_prot */ 102 uint32_t dir :1; /* SA direction */ 103 uint32_t resv0 :12; 104 uint32_t encap_dsbl:1; /* Encap/Decap disable */ 105 uint32_t resv1 :2; /* Must be set to 0 */ 106 } ctrl_word; 107 u32 spi; /* SPI Value */ 108 uint32_t pmtu_limit :16; /* PMTU Limit */ 109 uint32_t resv0 :5; 110 uint32_t ipv6 :1; /* Outbound IPv6 addr format */ 111 uint32_t resv1 :10; 112 u32 resv2[2]; 113 u32 src_ip[4]; /* Src IP addr */ 114 u32 dst_ip[4]; /* Dst IP addr */ 115 u32 resv3[6]; 116 }; 117 118 /* IPSEC_CFG_MSSG */ 119 struct nfp_ipsec_cfg_mssg { 120 union { 121 struct{ 122 uint32_t cmd:16; /* One of nfp_ipsec_cfg_mssg_cmd_codes */ 123 uint32_t rsp:16; /* One of nfp_ipsec_cfg_mssg_rsp_codes */ 124 uint32_t sa_idx:16; /* SA table index */ 125 uint32_t spare0:16; 126 struct nfp_ipsec_cfg_add_sa cfg_add_sa; 127 }; 128 u32 raw[64]; 129 }; 130 }; 131 132 static int nfp_ipsec_cfg_cmd_issue(struct nfp_net *nn, int type, int saidx, 133 struct nfp_ipsec_cfg_mssg *msg) 134 { 135 unsigned int offset = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL; 136 int i, msg_size, ret; 137 138 ret = nfp_net_mbox_lock(nn, sizeof(*msg)); 139 if (ret) 140 return ret; 141 142 msg->cmd = type; 143 msg->sa_idx = saidx; 144 msg->rsp = 0; 145 msg_size = ARRAY_SIZE(msg->raw); 146 147 for (i = 0; i < msg_size; i++) 148 nn_writel(nn, offset + 4 * i, msg->raw[i]); 149 150 ret = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_IPSEC); 151 if (ret < 0) { 152 nn_ctrl_bar_unlock(nn); 153 return ret; 154 } 155 156 /* For now we always read the whole message response back */ 157 for (i = 0; i < msg_size; i++) 158 msg->raw[i] = nn_readl(nn, offset + 4 * i); 159 160 nn_ctrl_bar_unlock(nn); 161 162 switch (msg->rsp) { 163 case NFP_IPSEC_CFG_MSSG_OK: 164 return 0; 165 case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD: 166 return -EINVAL; 167 case NFP_IPSEC_CFG_MSSG_SA_VALID: 168 return -EEXIST; 169 case NFP_IPSEC_CFG_MSSG_FAILED: 170 case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED: 171 case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED: 172 return -EIO; 173 default: 174 return -EINVAL; 175 } 176 } 177 178 static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen) 179 { 180 bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC); 181 182 switch (keylen) { 183 case 128: 184 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL : 185 NFP_IPSEC_CIPHER_AES128; 186 break; 187 case 192: 188 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL : 189 NFP_IPSEC_CIPHER_AES192; 190 break; 191 case 256: 192 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL : 193 NFP_IPSEC_CIPHER_AES256; 194 break; 195 default: 196 return -EINVAL; 197 } 198 199 return 0; 200 } 201 202 static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 203 { 204 switch (*trunc_len) { 205 case 96: 206 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96; 207 break; 208 case 128: 209 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128; 210 break; 211 default: 212 *trunc_len = 0; 213 } 214 } 215 216 static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 217 { 218 switch (*trunc_len) { 219 case 96: 220 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96; 221 break; 222 case 80: 223 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80; 224 break; 225 default: 226 *trunc_len = 0; 227 } 228 } 229 230 static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 231 { 232 switch (*trunc_len) { 233 case 96: 234 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96; 235 break; 236 case 128: 237 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128; 238 break; 239 default: 240 *trunc_len = 0; 241 } 242 } 243 244 static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 245 { 246 switch (*trunc_len) { 247 case 96: 248 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96; 249 break; 250 case 192: 251 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192; 252 break; 253 default: 254 *trunc_len = 0; 255 } 256 } 257 258 static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 259 { 260 switch (*trunc_len) { 261 case 96: 262 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96; 263 break; 264 case 256: 265 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256; 266 break; 267 default: 268 *trunc_len = 0; 269 } 270 } 271 272 static int nfp_net_xfrm_add_state(struct xfrm_state *x) 273 { 274 struct net_device *netdev = x->xso.dev; 275 struct nfp_ipsec_cfg_mssg msg = {}; 276 int i, key_len, trunc_len, err = 0; 277 struct nfp_ipsec_cfg_add_sa *cfg; 278 struct nfp_net *nn; 279 unsigned int saidx; 280 281 nn = netdev_priv(netdev); 282 cfg = &msg.cfg_add_sa; 283 284 /* General */ 285 switch (x->props.mode) { 286 case XFRM_MODE_TUNNEL: 287 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL; 288 break; 289 case XFRM_MODE_TRANSPORT: 290 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT; 291 break; 292 default: 293 nn_err(nn, "Unsupported mode for xfrm offload\n"); 294 return -EINVAL; 295 } 296 297 switch (x->id.proto) { 298 case IPPROTO_ESP: 299 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP; 300 break; 301 case IPPROTO_AH: 302 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH; 303 break; 304 default: 305 nn_err(nn, "Unsupported protocol for xfrm offload\n"); 306 return -EINVAL; 307 } 308 309 if (x->props.flags & XFRM_STATE_ESN) { 310 nn_err(nn, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload\n"); 311 return -EINVAL; 312 } 313 314 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { 315 nn_err(nn, "Unsupported xfrm offload tyoe\n"); 316 return -EINVAL; 317 } 318 319 cfg->spi = ntohl(x->id.spi); 320 321 /* Hash/Authentication */ 322 if (x->aalg) 323 trunc_len = x->aalg->alg_trunc_len; 324 else 325 trunc_len = 0; 326 327 switch (x->props.aalgo) { 328 case SADB_AALG_NONE: 329 if (x->aead) { 330 trunc_len = -1; 331 } else { 332 nn_err(nn, "Unsupported authentication algorithm\n"); 333 return -EINVAL; 334 } 335 break; 336 case SADB_X_AALG_NULL: 337 cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE; 338 trunc_len = -1; 339 break; 340 case SADB_AALG_MD5HMAC: 341 set_md5hmac(cfg, &trunc_len); 342 break; 343 case SADB_AALG_SHA1HMAC: 344 set_sha1hmac(cfg, &trunc_len); 345 break; 346 case SADB_X_AALG_SHA2_256HMAC: 347 set_sha2_256hmac(cfg, &trunc_len); 348 break; 349 case SADB_X_AALG_SHA2_384HMAC: 350 set_sha2_384hmac(cfg, &trunc_len); 351 break; 352 case SADB_X_AALG_SHA2_512HMAC: 353 set_sha2_512hmac(cfg, &trunc_len); 354 break; 355 default: 356 nn_err(nn, "Unsupported authentication algorithm\n"); 357 return -EINVAL; 358 } 359 360 if (!trunc_len) { 361 nn_err(nn, "Unsupported authentication algorithm trunc length\n"); 362 return -EINVAL; 363 } 364 365 if (x->aalg) { 366 key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE); 367 if (key_len > sizeof(cfg->auth_key)) { 368 nn_err(nn, "Insufficient space for offloaded auth key\n"); 369 return -EINVAL; 370 } 371 for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++) 372 cfg->auth_key[i] = get_unaligned_be32(x->aalg->alg_key + 373 sizeof(cfg->auth_key[0]) * i); 374 } 375 376 /* Encryption */ 377 switch (x->props.ealgo) { 378 case SADB_EALG_NONE: 379 case SADB_EALG_NULL: 380 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 381 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL; 382 break; 383 case SADB_EALG_3DESCBC: 384 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 385 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES; 386 break; 387 case SADB_X_EALG_AES_GCM_ICV16: 388 case SADB_X_EALG_NULL_AES_GMAC: 389 if (!x->aead) { 390 nn_err(nn, "Invalid AES key data\n"); 391 return -EINVAL; 392 } 393 394 if (x->aead->alg_icv_len != 128) { 395 nn_err(nn, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16\n"); 396 return -EINVAL; 397 } 398 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR; 399 cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128; 400 401 /* Aead->alg_key_len includes 32-bit salt */ 402 if (set_aes_keylen(cfg, x->props.ealgo, x->aead->alg_key_len - 32)) { 403 nn_err(nn, "Unsupported AES key length %d\n", x->aead->alg_key_len); 404 return -EINVAL; 405 } 406 break; 407 case SADB_X_EALG_AESCBC: 408 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 409 if (!x->ealg) { 410 nn_err(nn, "Invalid AES key data\n"); 411 return -EINVAL; 412 } 413 if (set_aes_keylen(cfg, x->props.ealgo, x->ealg->alg_key_len) < 0) { 414 nn_err(nn, "Unsupported AES key length %d\n", x->ealg->alg_key_len); 415 return -EINVAL; 416 } 417 break; 418 default: 419 nn_err(nn, "Unsupported encryption algorithm for offload\n"); 420 return -EINVAL; 421 } 422 423 if (x->aead) { 424 int salt_len = 4; 425 426 key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE); 427 key_len -= salt_len; 428 429 if (key_len > sizeof(cfg->ciph_key)) { 430 nn_err(nn, "aead: Insufficient space for offloaded key\n"); 431 return -EINVAL; 432 } 433 434 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 435 cfg->ciph_key[i] = get_unaligned_be32(x->aead->alg_key + 436 sizeof(cfg->ciph_key[0]) * i); 437 438 /* Load up the salt */ 439 cfg->aesgcm_fields.salt = get_unaligned_be32(x->aead->alg_key + key_len); 440 } 441 442 if (x->ealg) { 443 key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE); 444 445 if (key_len > sizeof(cfg->ciph_key)) { 446 nn_err(nn, "ealg: Insufficient space for offloaded key\n"); 447 return -EINVAL; 448 } 449 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 450 cfg->ciph_key[i] = get_unaligned_be32(x->ealg->alg_key + 451 sizeof(cfg->ciph_key[0]) * i); 452 } 453 454 /* IP related info */ 455 switch (x->props.family) { 456 case AF_INET: 457 cfg->ipv6 = 0; 458 cfg->src_ip[0] = ntohl(x->props.saddr.a4); 459 cfg->dst_ip[0] = ntohl(x->id.daddr.a4); 460 break; 461 case AF_INET6: 462 cfg->ipv6 = 1; 463 for (i = 0; i < 4; i++) { 464 cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]); 465 cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]); 466 } 467 break; 468 default: 469 nn_err(nn, "Unsupported address family\n"); 470 return -EINVAL; 471 } 472 473 /* Maximum nic IPsec code could handle. Other limits may apply. */ 474 cfg->pmtu_limit = 0xffff; 475 cfg->ctrl_word.encap_dsbl = 1; 476 477 /* SA direction */ 478 cfg->ctrl_word.dir = x->xso.dir; 479 480 /* Find unused SA data*/ 481 err = xa_alloc(&nn->xa_ipsec, &saidx, x, 482 XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL); 483 if (err < 0) { 484 nn_err(nn, "Unable to get sa_data number for IPsec\n"); 485 return err; 486 } 487 488 /* Allocate saidx and commit the SA */ 489 err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_ADD_SA, saidx, &msg); 490 if (err) { 491 xa_erase(&nn->xa_ipsec, saidx); 492 nn_err(nn, "Failed to issue IPsec command err ret=%d\n", err); 493 return err; 494 } 495 496 /* 0 is invalid offload_handle for kernel */ 497 x->xso.offload_handle = saidx + 1; 498 return 0; 499 } 500 501 static void nfp_net_xfrm_del_state(struct xfrm_state *x) 502 { 503 struct net_device *netdev = x->xso.dev; 504 struct nfp_ipsec_cfg_mssg msg; 505 struct nfp_net *nn; 506 int err; 507 508 nn = netdev_priv(netdev); 509 err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_INV_SA, 510 x->xso.offload_handle - 1, &msg); 511 if (err) 512 nn_warn(nn, "Failed to invalidate SA in hardware\n"); 513 514 xa_erase(&nn->xa_ipsec, x->xso.offload_handle - 1); 515 } 516 517 static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 518 { 519 if (x->props.family == AF_INET) 520 /* Offload with IPv4 options is not supported yet */ 521 return ip_hdr(skb)->ihl == 5; 522 523 /* Offload with IPv6 extension headers is not support yet */ 524 return !(ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)); 525 } 526 527 static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = { 528 .xdo_dev_state_add = nfp_net_xfrm_add_state, 529 .xdo_dev_state_delete = nfp_net_xfrm_del_state, 530 .xdo_dev_offload_ok = nfp_net_ipsec_offload_ok, 531 }; 532 533 void nfp_net_ipsec_init(struct nfp_net *nn) 534 { 535 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 536 return; 537 538 xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC); 539 nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops; 540 } 541 542 void nfp_net_ipsec_clean(struct nfp_net *nn) 543 { 544 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 545 return; 546 547 WARN_ON(!xa_empty(&nn->xa_ipsec)); 548 xa_destroy(&nn->xa_ipsec); 549 } 550 551 bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb, 552 struct nfp_ipsec_offload *offload_info) 553 { 554 struct xfrm_offload *xo = xfrm_offload(skb); 555 struct xfrm_state *x; 556 557 x = xfrm_input_state(skb); 558 if (!x) 559 return false; 560 561 offload_info->seq_hi = xo->seq.hi; 562 offload_info->seq_low = xo->seq.low; 563 offload_info->handle = x->xso.offload_handle; 564 565 return true; 566 } 567 568 int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb) 569 { 570 struct net_device *netdev = skb->dev; 571 struct xfrm_offload *xo; 572 struct xfrm_state *x; 573 struct sec_path *sp; 574 struct nfp_net *nn; 575 u32 saidx; 576 577 nn = netdev_priv(netdev); 578 579 saidx = meta->ipsec_saidx - 1; 580 if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT) 581 return -EINVAL; 582 583 sp = secpath_set(skb); 584 if (unlikely(!sp)) 585 return -ENOMEM; 586 587 xa_lock(&nn->xa_ipsec); 588 x = xa_load(&nn->xa_ipsec, saidx); 589 xa_unlock(&nn->xa_ipsec); 590 if (!x) 591 return -EINVAL; 592 593 xfrm_state_hold(x); 594 sp->xvec[sp->len++] = x; 595 sp->olen++; 596 xo = xfrm_offload(skb); 597 xo->flags = CRYPTO_DONE; 598 xo->status = CRYPTO_SUCCESS; 599 600 return 0; 601 } 602