1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc */ 3 /* Copyright (C) 2021 Corigine, Inc */ 4 5 #include <linux/module.h> 6 #include <linux/kernel.h> 7 #include <linux/init.h> 8 #include <linux/netdevice.h> 9 #include <asm/unaligned.h> 10 #include <linux/ktime.h> 11 #include <net/xfrm.h> 12 13 #include "../nfp_net_ctrl.h" 14 #include "../nfp_net.h" 15 #include "crypto.h" 16 17 #define NFP_NET_IPSEC_MAX_SA_CNT (16 * 1024) /* Firmware support a maximum of 16K SA offload */ 18 19 /* IPsec config message cmd codes */ 20 enum nfp_ipsec_cfg_mssg_cmd_codes { 21 NFP_IPSEC_CFG_MSSG_ADD_SA, /* Add a new SA */ 22 NFP_IPSEC_CFG_MSSG_INV_SA /* Invalidate an existing SA */ 23 }; 24 25 /* IPsec config message response codes */ 26 enum nfp_ipsec_cfg_mssg_rsp_codes { 27 NFP_IPSEC_CFG_MSSG_OK, 28 NFP_IPSEC_CFG_MSSG_FAILED, 29 NFP_IPSEC_CFG_MSSG_SA_VALID, 30 NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED, 31 NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED, 32 NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD 33 }; 34 35 /* Protocol */ 36 enum nfp_ipsec_sa_prot { 37 NFP_IPSEC_PROTOCOL_AH = 0, 38 NFP_IPSEC_PROTOCOL_ESP = 1 39 }; 40 41 /* Mode */ 42 enum nfp_ipsec_sa_mode { 43 NFP_IPSEC_PROTMODE_TRANSPORT = 0, 44 NFP_IPSEC_PROTMODE_TUNNEL = 1 45 }; 46 47 /* Cipher types */ 48 enum nfp_ipsec_sa_cipher { 49 NFP_IPSEC_CIPHER_NULL, 50 NFP_IPSEC_CIPHER_3DES, 51 NFP_IPSEC_CIPHER_AES128, 52 NFP_IPSEC_CIPHER_AES192, 53 NFP_IPSEC_CIPHER_AES256, 54 NFP_IPSEC_CIPHER_AES128_NULL, 55 NFP_IPSEC_CIPHER_AES192_NULL, 56 NFP_IPSEC_CIPHER_AES256_NULL, 57 NFP_IPSEC_CIPHER_CHACHA20 58 }; 59 60 /* Cipher modes */ 61 enum nfp_ipsec_sa_cipher_mode { 62 NFP_IPSEC_CIMODE_ECB, 63 NFP_IPSEC_CIMODE_CBC, 64 NFP_IPSEC_CIMODE_CFB, 65 NFP_IPSEC_CIMODE_OFB, 66 NFP_IPSEC_CIMODE_CTR 67 }; 68 69 /* Hash types */ 70 enum nfp_ipsec_sa_hash_type { 71 NFP_IPSEC_HASH_NONE, 72 NFP_IPSEC_HASH_MD5_96, 73 NFP_IPSEC_HASH_SHA1_96, 74 NFP_IPSEC_HASH_SHA256_96, 75 NFP_IPSEC_HASH_SHA384_96, 76 NFP_IPSEC_HASH_SHA512_96, 77 NFP_IPSEC_HASH_MD5_128, 78 NFP_IPSEC_HASH_SHA1_80, 79 NFP_IPSEC_HASH_SHA256_128, 80 NFP_IPSEC_HASH_SHA384_192, 81 NFP_IPSEC_HASH_SHA512_256, 82 NFP_IPSEC_HASH_GF128_128, 83 NFP_IPSEC_HASH_POLY1305_128 84 }; 85 86 /* IPSEC_CFG_MSSG_ADD_SA */ 87 struct nfp_ipsec_cfg_add_sa { 88 u32 ciph_key[8]; /* Cipher Key */ 89 union { 90 u32 auth_key[16]; /* Authentication Key */ 91 struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */ 92 u32 salt; /* Initialized with SA */ 93 u32 resv[15]; 94 } aesgcm_fields; 95 }; 96 struct sa_ctrl_word { 97 uint32_t hash :4; /* From nfp_ipsec_sa_hash_type */ 98 uint32_t cimode :4; /* From nfp_ipsec_sa_cipher_mode */ 99 uint32_t cipher :4; /* From nfp_ipsec_sa_cipher */ 100 uint32_t mode :2; /* From nfp_ipsec_sa_mode */ 101 uint32_t proto :2; /* From nfp_ipsec_sa_prot */ 102 uint32_t dir :1; /* SA direction */ 103 uint32_t resv0 :12; 104 uint32_t encap_dsbl:1; /* Encap/Decap disable */ 105 uint32_t resv1 :2; /* Must be set to 0 */ 106 } ctrl_word; 107 u32 spi; /* SPI Value */ 108 uint32_t pmtu_limit :16; /* PMTU Limit */ 109 uint32_t resv0 :5; 110 uint32_t ipv6 :1; /* Outbound IPv6 addr format */ 111 uint32_t resv1 :10; 112 u32 resv2[2]; 113 u32 src_ip[4]; /* Src IP addr */ 114 u32 dst_ip[4]; /* Dst IP addr */ 115 u32 resv3[6]; 116 }; 117 118 /* IPSEC_CFG_MSSG */ 119 struct nfp_ipsec_cfg_mssg { 120 union { 121 struct{ 122 uint32_t cmd:16; /* One of nfp_ipsec_cfg_mssg_cmd_codes */ 123 uint32_t rsp:16; /* One of nfp_ipsec_cfg_mssg_rsp_codes */ 124 uint32_t sa_idx:16; /* SA table index */ 125 uint32_t spare0:16; 126 struct nfp_ipsec_cfg_add_sa cfg_add_sa; 127 }; 128 u32 raw[64]; 129 }; 130 }; 131 132 static int nfp_ipsec_cfg_cmd_issue(struct nfp_net *nn, int type, int saidx, 133 struct nfp_ipsec_cfg_mssg *msg) 134 { 135 int i, msg_size, ret; 136 137 msg->cmd = type; 138 msg->sa_idx = saidx; 139 msg->rsp = 0; 140 msg_size = ARRAY_SIZE(msg->raw); 141 142 for (i = 0; i < msg_size; i++) 143 nn_writel(nn, NFP_NET_CFG_MBOX_VAL + 4 * i, msg->raw[i]); 144 145 ret = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_IPSEC); 146 if (ret < 0) 147 return ret; 148 149 /* For now we always read the whole message response back */ 150 for (i = 0; i < msg_size; i++) 151 msg->raw[i] = nn_readl(nn, NFP_NET_CFG_MBOX_VAL + 4 * i); 152 153 switch (msg->rsp) { 154 case NFP_IPSEC_CFG_MSSG_OK: 155 return 0; 156 case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD: 157 return -EINVAL; 158 case NFP_IPSEC_CFG_MSSG_SA_VALID: 159 return -EEXIST; 160 case NFP_IPSEC_CFG_MSSG_FAILED: 161 case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED: 162 case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED: 163 return -EIO; 164 default: 165 return -EINVAL; 166 } 167 } 168 169 static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen) 170 { 171 bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC); 172 173 switch (keylen) { 174 case 128: 175 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL : 176 NFP_IPSEC_CIPHER_AES128; 177 break; 178 case 192: 179 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL : 180 NFP_IPSEC_CIPHER_AES192; 181 break; 182 case 256: 183 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL : 184 NFP_IPSEC_CIPHER_AES256; 185 break; 186 default: 187 return -EINVAL; 188 } 189 190 return 0; 191 } 192 193 static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 194 { 195 switch (*trunc_len) { 196 case 96: 197 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96; 198 break; 199 case 128: 200 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128; 201 break; 202 default: 203 *trunc_len = 0; 204 } 205 } 206 207 static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 208 { 209 switch (*trunc_len) { 210 case 96: 211 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96; 212 break; 213 case 80: 214 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80; 215 break; 216 default: 217 *trunc_len = 0; 218 } 219 } 220 221 static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 222 { 223 switch (*trunc_len) { 224 case 96: 225 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96; 226 break; 227 case 128: 228 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128; 229 break; 230 default: 231 *trunc_len = 0; 232 } 233 } 234 235 static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 236 { 237 switch (*trunc_len) { 238 case 96: 239 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96; 240 break; 241 case 192: 242 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192; 243 break; 244 default: 245 *trunc_len = 0; 246 } 247 } 248 249 static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 250 { 251 switch (*trunc_len) { 252 case 96: 253 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96; 254 break; 255 case 256: 256 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256; 257 break; 258 default: 259 *trunc_len = 0; 260 } 261 } 262 263 static int nfp_net_xfrm_add_state(struct xfrm_state *x) 264 { 265 struct net_device *netdev = x->xso.dev; 266 struct nfp_ipsec_cfg_mssg msg = {}; 267 int i, key_len, trunc_len, err = 0; 268 struct nfp_ipsec_cfg_add_sa *cfg; 269 struct nfp_net *nn; 270 unsigned int saidx; 271 272 nn = netdev_priv(netdev); 273 cfg = &msg.cfg_add_sa; 274 275 /* General */ 276 switch (x->props.mode) { 277 case XFRM_MODE_TUNNEL: 278 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL; 279 break; 280 case XFRM_MODE_TRANSPORT: 281 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT; 282 break; 283 default: 284 nn_err(nn, "Unsupported mode for xfrm offload\n"); 285 return -EINVAL; 286 } 287 288 switch (x->id.proto) { 289 case IPPROTO_ESP: 290 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP; 291 break; 292 case IPPROTO_AH: 293 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH; 294 break; 295 default: 296 nn_err(nn, "Unsupported protocol for xfrm offload\n"); 297 return -EINVAL; 298 } 299 300 if (x->props.flags & XFRM_STATE_ESN) { 301 nn_err(nn, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload\n"); 302 return -EINVAL; 303 } 304 305 cfg->spi = ntohl(x->id.spi); 306 307 /* Hash/Authentication */ 308 if (x->aalg) 309 trunc_len = x->aalg->alg_trunc_len; 310 else 311 trunc_len = 0; 312 313 switch (x->props.aalgo) { 314 case SADB_AALG_NONE: 315 if (x->aead) { 316 trunc_len = -1; 317 } else { 318 nn_err(nn, "Unsupported authentication algorithm\n"); 319 return -EINVAL; 320 } 321 break; 322 case SADB_X_AALG_NULL: 323 cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE; 324 trunc_len = -1; 325 break; 326 case SADB_AALG_MD5HMAC: 327 set_md5hmac(cfg, &trunc_len); 328 break; 329 case SADB_AALG_SHA1HMAC: 330 set_sha1hmac(cfg, &trunc_len); 331 break; 332 case SADB_X_AALG_SHA2_256HMAC: 333 set_sha2_256hmac(cfg, &trunc_len); 334 break; 335 case SADB_X_AALG_SHA2_384HMAC: 336 set_sha2_384hmac(cfg, &trunc_len); 337 break; 338 case SADB_X_AALG_SHA2_512HMAC: 339 set_sha2_512hmac(cfg, &trunc_len); 340 break; 341 default: 342 nn_err(nn, "Unsupported authentication algorithm\n"); 343 return -EINVAL; 344 } 345 346 if (!trunc_len) { 347 nn_err(nn, "Unsupported authentication algorithm trunc length\n"); 348 return -EINVAL; 349 } 350 351 if (x->aalg) { 352 key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE); 353 if (key_len > sizeof(cfg->auth_key)) { 354 nn_err(nn, "Insufficient space for offloaded auth key\n"); 355 return -EINVAL; 356 } 357 for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++) 358 cfg->auth_key[i] = get_unaligned_be32(x->aalg->alg_key + 359 sizeof(cfg->auth_key[0]) * i); 360 } 361 362 /* Encryption */ 363 switch (x->props.ealgo) { 364 case SADB_EALG_NONE: 365 case SADB_EALG_NULL: 366 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 367 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL; 368 break; 369 case SADB_EALG_3DESCBC: 370 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 371 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES; 372 break; 373 case SADB_X_EALG_AES_GCM_ICV16: 374 case SADB_X_EALG_NULL_AES_GMAC: 375 if (!x->aead) { 376 nn_err(nn, "Invalid AES key data\n"); 377 return -EINVAL; 378 } 379 380 if (x->aead->alg_icv_len != 128) { 381 nn_err(nn, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16\n"); 382 return -EINVAL; 383 } 384 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR; 385 cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128; 386 387 /* Aead->alg_key_len includes 32-bit salt */ 388 if (set_aes_keylen(cfg, x->props.ealgo, x->aead->alg_key_len - 32)) { 389 nn_err(nn, "Unsupported AES key length %d\n", x->aead->alg_key_len); 390 return -EINVAL; 391 } 392 break; 393 case SADB_X_EALG_AESCBC: 394 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 395 if (!x->ealg) { 396 nn_err(nn, "Invalid AES key data\n"); 397 return -EINVAL; 398 } 399 if (set_aes_keylen(cfg, x->props.ealgo, x->ealg->alg_key_len) < 0) { 400 nn_err(nn, "Unsupported AES key length %d\n", x->ealg->alg_key_len); 401 return -EINVAL; 402 } 403 break; 404 default: 405 nn_err(nn, "Unsupported encryption algorithm for offload\n"); 406 return -EINVAL; 407 } 408 409 if (x->aead) { 410 int salt_len = 4; 411 412 key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE); 413 key_len -= salt_len; 414 415 if (key_len > sizeof(cfg->ciph_key)) { 416 nn_err(nn, "aead: Insufficient space for offloaded key\n"); 417 return -EINVAL; 418 } 419 420 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 421 cfg->ciph_key[i] = get_unaligned_be32(x->aead->alg_key + 422 sizeof(cfg->ciph_key[0]) * i); 423 424 /* Load up the salt */ 425 cfg->aesgcm_fields.salt = get_unaligned_be32(x->aead->alg_key + key_len); 426 } 427 428 if (x->ealg) { 429 key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE); 430 431 if (key_len > sizeof(cfg->ciph_key)) { 432 nn_err(nn, "ealg: Insufficient space for offloaded key\n"); 433 return -EINVAL; 434 } 435 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 436 cfg->ciph_key[i] = get_unaligned_be32(x->ealg->alg_key + 437 sizeof(cfg->ciph_key[0]) * i); 438 } 439 440 /* IP related info */ 441 switch (x->props.family) { 442 case AF_INET: 443 cfg->ipv6 = 0; 444 cfg->src_ip[0] = ntohl(x->props.saddr.a4); 445 cfg->dst_ip[0] = ntohl(x->id.daddr.a4); 446 break; 447 case AF_INET6: 448 cfg->ipv6 = 1; 449 for (i = 0; i < 4; i++) { 450 cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]); 451 cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]); 452 } 453 break; 454 default: 455 nn_err(nn, "Unsupported address family\n"); 456 return -EINVAL; 457 } 458 459 /* Maximum nic IPsec code could handle. Other limits may apply. */ 460 cfg->pmtu_limit = 0xffff; 461 cfg->ctrl_word.encap_dsbl = 1; 462 463 /* SA direction */ 464 cfg->ctrl_word.dir = x->xso.dir; 465 466 /* Find unused SA data*/ 467 err = xa_alloc(&nn->xa_ipsec, &saidx, x, 468 XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL); 469 if (err < 0) { 470 nn_err(nn, "Unable to get sa_data number for IPsec\n"); 471 return err; 472 } 473 474 /* Allocate saidx and commit the SA */ 475 err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_ADD_SA, saidx, &msg); 476 if (err) { 477 xa_erase(&nn->xa_ipsec, saidx); 478 nn_err(nn, "Failed to issue IPsec command err ret=%d\n", err); 479 return err; 480 } 481 482 /* 0 is invalid offload_handle for kernel */ 483 x->xso.offload_handle = saidx + 1; 484 return 0; 485 } 486 487 static void nfp_net_xfrm_del_state(struct xfrm_state *x) 488 { 489 struct net_device *netdev = x->xso.dev; 490 struct nfp_ipsec_cfg_mssg msg; 491 struct nfp_net *nn; 492 int err; 493 494 nn = netdev_priv(netdev); 495 err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_INV_SA, 496 x->xso.offload_handle - 1, &msg); 497 if (err) 498 nn_warn(nn, "Failed to invalidate SA in hardware\n"); 499 500 xa_erase(&nn->xa_ipsec, x->xso.offload_handle - 1); 501 } 502 503 static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 504 { 505 if (x->props.family == AF_INET) 506 /* Offload with IPv4 options is not supported yet */ 507 return ip_hdr(skb)->ihl == 5; 508 509 /* Offload with IPv6 extension headers is not support yet */ 510 return !(ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)); 511 } 512 513 static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = { 514 .xdo_dev_state_add = nfp_net_xfrm_add_state, 515 .xdo_dev_state_delete = nfp_net_xfrm_del_state, 516 .xdo_dev_offload_ok = nfp_net_ipsec_offload_ok, 517 }; 518 519 void nfp_net_ipsec_init(struct nfp_net *nn) 520 { 521 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 522 return; 523 524 xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC); 525 nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops; 526 } 527 528 void nfp_net_ipsec_clean(struct nfp_net *nn) 529 { 530 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 531 return; 532 533 WARN_ON(!xa_empty(&nn->xa_ipsec)); 534 xa_destroy(&nn->xa_ipsec); 535 } 536 537 bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb, 538 struct nfp_ipsec_offload *offload_info) 539 { 540 struct xfrm_offload *xo = xfrm_offload(skb); 541 struct xfrm_state *x; 542 543 x = xfrm_input_state(skb); 544 if (!x) 545 return false; 546 547 offload_info->seq_hi = xo->seq.hi; 548 offload_info->seq_low = xo->seq.low; 549 offload_info->handle = x->xso.offload_handle; 550 551 return true; 552 } 553 554 int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb) 555 { 556 struct net_device *netdev = skb->dev; 557 struct xfrm_offload *xo; 558 struct xfrm_state *x; 559 struct sec_path *sp; 560 struct nfp_net *nn; 561 u32 saidx; 562 563 nn = netdev_priv(netdev); 564 565 saidx = meta->ipsec_saidx - 1; 566 if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT) 567 return -EINVAL; 568 569 sp = secpath_set(skb); 570 if (unlikely(!sp)) 571 return -ENOMEM; 572 573 xa_lock(&nn->xa_ipsec); 574 x = xa_load(&nn->xa_ipsec, saidx); 575 xa_unlock(&nn->xa_ipsec); 576 if (!x) 577 return -EINVAL; 578 579 xfrm_state_hold(x); 580 sp->xvec[sp->len++] = x; 581 sp->olen++; 582 xo = xfrm_offload(skb); 583 xo->flags = CRYPTO_DONE; 584 xo->status = CRYPTO_SUCCESS; 585 586 return 0; 587 } 588