1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc */ 3 /* Copyright (C) 2021 Corigine, Inc */ 4 5 #include <linux/module.h> 6 #include <linux/kernel.h> 7 #include <linux/init.h> 8 #include <linux/netdevice.h> 9 #include <asm/unaligned.h> 10 #include <linux/ktime.h> 11 #include <net/xfrm.h> 12 13 #include "../nfp_net_ctrl.h" 14 #include "../nfp_net.h" 15 #include "crypto.h" 16 17 #define NFP_NET_IPSEC_MAX_SA_CNT (16 * 1024) /* Firmware support a maximum of 16K SA offload */ 18 19 /* IPsec config message cmd codes */ 20 enum nfp_ipsec_cfg_mssg_cmd_codes { 21 NFP_IPSEC_CFG_MSSG_ADD_SA, /* Add a new SA */ 22 NFP_IPSEC_CFG_MSSG_INV_SA /* Invalidate an existing SA */ 23 }; 24 25 /* IPsec config message response codes */ 26 enum nfp_ipsec_cfg_mssg_rsp_codes { 27 NFP_IPSEC_CFG_MSSG_OK, 28 NFP_IPSEC_CFG_MSSG_FAILED, 29 NFP_IPSEC_CFG_MSSG_SA_VALID, 30 NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED, 31 NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED, 32 NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD 33 }; 34 35 /* Protocol */ 36 enum nfp_ipsec_sa_prot { 37 NFP_IPSEC_PROTOCOL_AH = 0, 38 NFP_IPSEC_PROTOCOL_ESP = 1 39 }; 40 41 /* Mode */ 42 enum nfp_ipsec_sa_mode { 43 NFP_IPSEC_PROTMODE_TRANSPORT = 0, 44 NFP_IPSEC_PROTMODE_TUNNEL = 1 45 }; 46 47 /* Cipher types */ 48 enum nfp_ipsec_sa_cipher { 49 NFP_IPSEC_CIPHER_NULL, 50 NFP_IPSEC_CIPHER_3DES, 51 NFP_IPSEC_CIPHER_AES128, 52 NFP_IPSEC_CIPHER_AES192, 53 NFP_IPSEC_CIPHER_AES256, 54 NFP_IPSEC_CIPHER_AES128_NULL, 55 NFP_IPSEC_CIPHER_AES192_NULL, 56 NFP_IPSEC_CIPHER_AES256_NULL, 57 NFP_IPSEC_CIPHER_CHACHA20 58 }; 59 60 /* Cipher modes */ 61 enum nfp_ipsec_sa_cipher_mode { 62 NFP_IPSEC_CIMODE_ECB, 63 NFP_IPSEC_CIMODE_CBC, 64 NFP_IPSEC_CIMODE_CFB, 65 NFP_IPSEC_CIMODE_OFB, 66 NFP_IPSEC_CIMODE_CTR 67 }; 68 69 /* Hash types */ 70 enum nfp_ipsec_sa_hash_type { 71 NFP_IPSEC_HASH_NONE, 72 NFP_IPSEC_HASH_MD5_96, 73 NFP_IPSEC_HASH_SHA1_96, 74 NFP_IPSEC_HASH_SHA256_96, 75 NFP_IPSEC_HASH_SHA384_96, 76 NFP_IPSEC_HASH_SHA512_96, 77 NFP_IPSEC_HASH_MD5_128, 78 NFP_IPSEC_HASH_SHA1_80, 79 NFP_IPSEC_HASH_SHA256_128, 80 NFP_IPSEC_HASH_SHA384_192, 81 NFP_IPSEC_HASH_SHA512_256, 82 NFP_IPSEC_HASH_GF128_128, 83 NFP_IPSEC_HASH_POLY1305_128 84 }; 85 86 /* IPSEC_CFG_MSSG_ADD_SA */ 87 struct nfp_ipsec_cfg_add_sa { 88 u32 ciph_key[8]; /* Cipher Key */ 89 union { 90 u32 auth_key[16]; /* Authentication Key */ 91 struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */ 92 u32 salt; /* Initialized with SA */ 93 u32 resv[15]; 94 } aesgcm_fields; 95 }; 96 struct sa_ctrl_word { 97 uint32_t hash :4; /* From nfp_ipsec_sa_hash_type */ 98 uint32_t cimode :4; /* From nfp_ipsec_sa_cipher_mode */ 99 uint32_t cipher :4; /* From nfp_ipsec_sa_cipher */ 100 uint32_t mode :2; /* From nfp_ipsec_sa_mode */ 101 uint32_t proto :2; /* From nfp_ipsec_sa_prot */ 102 uint32_t dir :1; /* SA direction */ 103 uint32_t resv0 :12; 104 uint32_t encap_dsbl:1; /* Encap/Decap disable */ 105 uint32_t resv1 :2; /* Must be set to 0 */ 106 } ctrl_word; 107 u32 spi; /* SPI Value */ 108 uint32_t pmtu_limit :16; /* PMTU Limit */ 109 uint32_t resv0 :5; 110 uint32_t ipv6 :1; /* Outbound IPv6 addr format */ 111 uint32_t resv1 :10; 112 u32 resv2[2]; 113 u32 src_ip[4]; /* Src IP addr */ 114 u32 dst_ip[4]; /* Dst IP addr */ 115 u32 resv3[6]; 116 }; 117 118 /* IPSEC_CFG_MSSG */ 119 struct nfp_ipsec_cfg_mssg { 120 union { 121 struct{ 122 uint32_t cmd:16; /* One of nfp_ipsec_cfg_mssg_cmd_codes */ 123 uint32_t rsp:16; /* One of nfp_ipsec_cfg_mssg_rsp_codes */ 124 uint32_t sa_idx:16; /* SA table index */ 125 uint32_t spare0:16; 126 struct nfp_ipsec_cfg_add_sa cfg_add_sa; 127 }; 128 u32 raw[64]; 129 }; 130 }; 131 132 static int nfp_ipsec_cfg_cmd_issue(struct nfp_net *nn, int type, int saidx, 133 struct nfp_ipsec_cfg_mssg *msg) 134 { 135 int i, msg_size, ret; 136 137 msg->cmd = type; 138 msg->sa_idx = saidx; 139 msg->rsp = 0; 140 msg_size = ARRAY_SIZE(msg->raw); 141 142 for (i = 0; i < msg_size; i++) 143 nn_writel(nn, NFP_NET_CFG_MBOX_VAL + 4 * i, msg->raw[i]); 144 145 ret = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_IPSEC); 146 if (ret < 0) 147 return ret; 148 149 /* For now we always read the whole message response back */ 150 for (i = 0; i < msg_size; i++) 151 msg->raw[i] = nn_readl(nn, NFP_NET_CFG_MBOX_VAL + 4 * i); 152 153 switch (msg->rsp) { 154 case NFP_IPSEC_CFG_MSSG_OK: 155 return 0; 156 case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD: 157 return -EINVAL; 158 case NFP_IPSEC_CFG_MSSG_SA_VALID: 159 return -EEXIST; 160 case NFP_IPSEC_CFG_MSSG_FAILED: 161 case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED: 162 case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED: 163 return -EIO; 164 default: 165 return -EINVAL; 166 } 167 } 168 169 static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen) 170 { 171 bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC); 172 173 switch (keylen) { 174 case 128: 175 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL : 176 NFP_IPSEC_CIPHER_AES128; 177 break; 178 case 192: 179 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL : 180 NFP_IPSEC_CIPHER_AES192; 181 break; 182 case 256: 183 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL : 184 NFP_IPSEC_CIPHER_AES256; 185 break; 186 default: 187 return -EINVAL; 188 } 189 190 return 0; 191 } 192 193 static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 194 { 195 switch (*trunc_len) { 196 case 96: 197 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96; 198 break; 199 case 128: 200 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128; 201 break; 202 default: 203 *trunc_len = 0; 204 } 205 } 206 207 static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 208 { 209 switch (*trunc_len) { 210 case 96: 211 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96; 212 break; 213 case 80: 214 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80; 215 break; 216 default: 217 *trunc_len = 0; 218 } 219 } 220 221 static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 222 { 223 switch (*trunc_len) { 224 case 96: 225 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96; 226 break; 227 case 128: 228 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128; 229 break; 230 default: 231 *trunc_len = 0; 232 } 233 } 234 235 static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 236 { 237 switch (*trunc_len) { 238 case 96: 239 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96; 240 break; 241 case 192: 242 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192; 243 break; 244 default: 245 *trunc_len = 0; 246 } 247 } 248 249 static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 250 { 251 switch (*trunc_len) { 252 case 96: 253 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96; 254 break; 255 case 256: 256 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256; 257 break; 258 default: 259 *trunc_len = 0; 260 } 261 } 262 263 static int nfp_net_xfrm_add_state(struct xfrm_state *x) 264 { 265 struct net_device *netdev = x->xso.dev; 266 struct nfp_ipsec_cfg_mssg msg = {}; 267 int i, key_len, trunc_len, err = 0; 268 struct nfp_ipsec_cfg_add_sa *cfg; 269 struct nfp_net *nn; 270 unsigned int saidx; 271 272 nn = netdev_priv(netdev); 273 cfg = &msg.cfg_add_sa; 274 275 /* General */ 276 switch (x->props.mode) { 277 case XFRM_MODE_TUNNEL: 278 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL; 279 break; 280 case XFRM_MODE_TRANSPORT: 281 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT; 282 break; 283 default: 284 nn_err(nn, "Unsupported mode for xfrm offload\n"); 285 return -EINVAL; 286 } 287 288 switch (x->id.proto) { 289 case IPPROTO_ESP: 290 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP; 291 break; 292 case IPPROTO_AH: 293 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH; 294 break; 295 default: 296 nn_err(nn, "Unsupported protocol for xfrm offload\n"); 297 return -EINVAL; 298 } 299 300 if (x->props.flags & XFRM_STATE_ESN) { 301 nn_err(nn, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload\n"); 302 return -EINVAL; 303 } 304 305 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { 306 nn_err(nn, "Unsupported xfrm offload tyoe\n"); 307 return -EINVAL; 308 } 309 310 cfg->spi = ntohl(x->id.spi); 311 312 /* Hash/Authentication */ 313 if (x->aalg) 314 trunc_len = x->aalg->alg_trunc_len; 315 else 316 trunc_len = 0; 317 318 switch (x->props.aalgo) { 319 case SADB_AALG_NONE: 320 if (x->aead) { 321 trunc_len = -1; 322 } else { 323 nn_err(nn, "Unsupported authentication algorithm\n"); 324 return -EINVAL; 325 } 326 break; 327 case SADB_X_AALG_NULL: 328 cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE; 329 trunc_len = -1; 330 break; 331 case SADB_AALG_MD5HMAC: 332 set_md5hmac(cfg, &trunc_len); 333 break; 334 case SADB_AALG_SHA1HMAC: 335 set_sha1hmac(cfg, &trunc_len); 336 break; 337 case SADB_X_AALG_SHA2_256HMAC: 338 set_sha2_256hmac(cfg, &trunc_len); 339 break; 340 case SADB_X_AALG_SHA2_384HMAC: 341 set_sha2_384hmac(cfg, &trunc_len); 342 break; 343 case SADB_X_AALG_SHA2_512HMAC: 344 set_sha2_512hmac(cfg, &trunc_len); 345 break; 346 default: 347 nn_err(nn, "Unsupported authentication algorithm\n"); 348 return -EINVAL; 349 } 350 351 if (!trunc_len) { 352 nn_err(nn, "Unsupported authentication algorithm trunc length\n"); 353 return -EINVAL; 354 } 355 356 if (x->aalg) { 357 key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE); 358 if (key_len > sizeof(cfg->auth_key)) { 359 nn_err(nn, "Insufficient space for offloaded auth key\n"); 360 return -EINVAL; 361 } 362 for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++) 363 cfg->auth_key[i] = get_unaligned_be32(x->aalg->alg_key + 364 sizeof(cfg->auth_key[0]) * i); 365 } 366 367 /* Encryption */ 368 switch (x->props.ealgo) { 369 case SADB_EALG_NONE: 370 case SADB_EALG_NULL: 371 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 372 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL; 373 break; 374 case SADB_EALG_3DESCBC: 375 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 376 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES; 377 break; 378 case SADB_X_EALG_AES_GCM_ICV16: 379 case SADB_X_EALG_NULL_AES_GMAC: 380 if (!x->aead) { 381 nn_err(nn, "Invalid AES key data\n"); 382 return -EINVAL; 383 } 384 385 if (x->aead->alg_icv_len != 128) { 386 nn_err(nn, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16\n"); 387 return -EINVAL; 388 } 389 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR; 390 cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128; 391 392 /* Aead->alg_key_len includes 32-bit salt */ 393 if (set_aes_keylen(cfg, x->props.ealgo, x->aead->alg_key_len - 32)) { 394 nn_err(nn, "Unsupported AES key length %d\n", x->aead->alg_key_len); 395 return -EINVAL; 396 } 397 break; 398 case SADB_X_EALG_AESCBC: 399 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 400 if (!x->ealg) { 401 nn_err(nn, "Invalid AES key data\n"); 402 return -EINVAL; 403 } 404 if (set_aes_keylen(cfg, x->props.ealgo, x->ealg->alg_key_len) < 0) { 405 nn_err(nn, "Unsupported AES key length %d\n", x->ealg->alg_key_len); 406 return -EINVAL; 407 } 408 break; 409 default: 410 nn_err(nn, "Unsupported encryption algorithm for offload\n"); 411 return -EINVAL; 412 } 413 414 if (x->aead) { 415 int salt_len = 4; 416 417 key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE); 418 key_len -= salt_len; 419 420 if (key_len > sizeof(cfg->ciph_key)) { 421 nn_err(nn, "aead: Insufficient space for offloaded key\n"); 422 return -EINVAL; 423 } 424 425 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 426 cfg->ciph_key[i] = get_unaligned_be32(x->aead->alg_key + 427 sizeof(cfg->ciph_key[0]) * i); 428 429 /* Load up the salt */ 430 cfg->aesgcm_fields.salt = get_unaligned_be32(x->aead->alg_key + key_len); 431 } 432 433 if (x->ealg) { 434 key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE); 435 436 if (key_len > sizeof(cfg->ciph_key)) { 437 nn_err(nn, "ealg: Insufficient space for offloaded key\n"); 438 return -EINVAL; 439 } 440 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 441 cfg->ciph_key[i] = get_unaligned_be32(x->ealg->alg_key + 442 sizeof(cfg->ciph_key[0]) * i); 443 } 444 445 /* IP related info */ 446 switch (x->props.family) { 447 case AF_INET: 448 cfg->ipv6 = 0; 449 cfg->src_ip[0] = ntohl(x->props.saddr.a4); 450 cfg->dst_ip[0] = ntohl(x->id.daddr.a4); 451 break; 452 case AF_INET6: 453 cfg->ipv6 = 1; 454 for (i = 0; i < 4; i++) { 455 cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]); 456 cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]); 457 } 458 break; 459 default: 460 nn_err(nn, "Unsupported address family\n"); 461 return -EINVAL; 462 } 463 464 /* Maximum nic IPsec code could handle. Other limits may apply. */ 465 cfg->pmtu_limit = 0xffff; 466 cfg->ctrl_word.encap_dsbl = 1; 467 468 /* SA direction */ 469 cfg->ctrl_word.dir = x->xso.dir; 470 471 /* Find unused SA data*/ 472 err = xa_alloc(&nn->xa_ipsec, &saidx, x, 473 XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL); 474 if (err < 0) { 475 nn_err(nn, "Unable to get sa_data number for IPsec\n"); 476 return err; 477 } 478 479 /* Allocate saidx and commit the SA */ 480 err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_ADD_SA, saidx, &msg); 481 if (err) { 482 xa_erase(&nn->xa_ipsec, saidx); 483 nn_err(nn, "Failed to issue IPsec command err ret=%d\n", err); 484 return err; 485 } 486 487 /* 0 is invalid offload_handle for kernel */ 488 x->xso.offload_handle = saidx + 1; 489 return 0; 490 } 491 492 static void nfp_net_xfrm_del_state(struct xfrm_state *x) 493 { 494 struct net_device *netdev = x->xso.dev; 495 struct nfp_ipsec_cfg_mssg msg; 496 struct nfp_net *nn; 497 int err; 498 499 nn = netdev_priv(netdev); 500 err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_INV_SA, 501 x->xso.offload_handle - 1, &msg); 502 if (err) 503 nn_warn(nn, "Failed to invalidate SA in hardware\n"); 504 505 xa_erase(&nn->xa_ipsec, x->xso.offload_handle - 1); 506 } 507 508 static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 509 { 510 if (x->props.family == AF_INET) 511 /* Offload with IPv4 options is not supported yet */ 512 return ip_hdr(skb)->ihl == 5; 513 514 /* Offload with IPv6 extension headers is not support yet */ 515 return !(ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)); 516 } 517 518 static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = { 519 .xdo_dev_state_add = nfp_net_xfrm_add_state, 520 .xdo_dev_state_delete = nfp_net_xfrm_del_state, 521 .xdo_dev_offload_ok = nfp_net_ipsec_offload_ok, 522 }; 523 524 void nfp_net_ipsec_init(struct nfp_net *nn) 525 { 526 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 527 return; 528 529 xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC); 530 nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops; 531 } 532 533 void nfp_net_ipsec_clean(struct nfp_net *nn) 534 { 535 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 536 return; 537 538 WARN_ON(!xa_empty(&nn->xa_ipsec)); 539 xa_destroy(&nn->xa_ipsec); 540 } 541 542 bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb, 543 struct nfp_ipsec_offload *offload_info) 544 { 545 struct xfrm_offload *xo = xfrm_offload(skb); 546 struct xfrm_state *x; 547 548 x = xfrm_input_state(skb); 549 if (!x) 550 return false; 551 552 offload_info->seq_hi = xo->seq.hi; 553 offload_info->seq_low = xo->seq.low; 554 offload_info->handle = x->xso.offload_handle; 555 556 return true; 557 } 558 559 int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb) 560 { 561 struct net_device *netdev = skb->dev; 562 struct xfrm_offload *xo; 563 struct xfrm_state *x; 564 struct sec_path *sp; 565 struct nfp_net *nn; 566 u32 saidx; 567 568 nn = netdev_priv(netdev); 569 570 saidx = meta->ipsec_saidx - 1; 571 if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT) 572 return -EINVAL; 573 574 sp = secpath_set(skb); 575 if (unlikely(!sp)) 576 return -ENOMEM; 577 578 xa_lock(&nn->xa_ipsec); 579 x = xa_load(&nn->xa_ipsec, saidx); 580 xa_unlock(&nn->xa_ipsec); 581 if (!x) 582 return -EINVAL; 583 584 xfrm_state_hold(x); 585 sp->xvec[sp->len++] = x; 586 sp->olen++; 587 xo = xfrm_offload(skb); 588 xo->flags = CRYPTO_DONE; 589 xo->status = CRYPTO_SUCCESS; 590 591 return 0; 592 } 593