1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc */ 3 /* Copyright (C) 2021 Corigine, Inc */ 4 5 #include <linux/module.h> 6 #include <linux/kernel.h> 7 #include <linux/init.h> 8 #include <linux/netdevice.h> 9 #include <asm/unaligned.h> 10 #include <linux/ktime.h> 11 #include <net/xfrm.h> 12 13 #include "../nfpcore/nfp_dev.h" 14 #include "../nfp_net_ctrl.h" 15 #include "../nfp_net.h" 16 #include "crypto.h" 17 18 #define NFP_NET_IPSEC_MAX_SA_CNT (16 * 1024) /* Firmware support a maximum of 16K SA offload */ 19 20 /* IPsec config message cmd codes */ 21 enum nfp_ipsec_cfg_mssg_cmd_codes { 22 NFP_IPSEC_CFG_MSSG_ADD_SA, /* Add a new SA */ 23 NFP_IPSEC_CFG_MSSG_INV_SA /* Invalidate an existing SA */ 24 }; 25 26 /* IPsec config message response codes */ 27 enum nfp_ipsec_cfg_mssg_rsp_codes { 28 NFP_IPSEC_CFG_MSSG_OK, 29 NFP_IPSEC_CFG_MSSG_FAILED, 30 NFP_IPSEC_CFG_MSSG_SA_VALID, 31 NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED, 32 NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED, 33 NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD 34 }; 35 36 /* Protocol */ 37 enum nfp_ipsec_sa_prot { 38 NFP_IPSEC_PROTOCOL_AH = 0, 39 NFP_IPSEC_PROTOCOL_ESP = 1 40 }; 41 42 /* Mode */ 43 enum nfp_ipsec_sa_mode { 44 NFP_IPSEC_PROTMODE_TRANSPORT = 0, 45 NFP_IPSEC_PROTMODE_TUNNEL = 1 46 }; 47 48 /* Cipher types */ 49 enum nfp_ipsec_sa_cipher { 50 NFP_IPSEC_CIPHER_NULL, 51 NFP_IPSEC_CIPHER_3DES, 52 NFP_IPSEC_CIPHER_AES128, 53 NFP_IPSEC_CIPHER_AES192, 54 NFP_IPSEC_CIPHER_AES256, 55 NFP_IPSEC_CIPHER_AES128_NULL, 56 NFP_IPSEC_CIPHER_AES192_NULL, 57 NFP_IPSEC_CIPHER_AES256_NULL, 58 NFP_IPSEC_CIPHER_CHACHA20 59 }; 60 61 /* Cipher modes */ 62 enum nfp_ipsec_sa_cipher_mode { 63 NFP_IPSEC_CIMODE_ECB, 64 NFP_IPSEC_CIMODE_CBC, 65 NFP_IPSEC_CIMODE_CFB, 66 NFP_IPSEC_CIMODE_OFB, 67 NFP_IPSEC_CIMODE_CTR 68 }; 69 70 /* Hash types */ 71 enum nfp_ipsec_sa_hash_type { 72 NFP_IPSEC_HASH_NONE, 73 NFP_IPSEC_HASH_MD5_96, 74 NFP_IPSEC_HASH_SHA1_96, 75 NFP_IPSEC_HASH_SHA256_96, 76 NFP_IPSEC_HASH_SHA384_96, 77 NFP_IPSEC_HASH_SHA512_96, 78 NFP_IPSEC_HASH_MD5_128, 79 NFP_IPSEC_HASH_SHA1_80, 80 NFP_IPSEC_HASH_SHA256_128, 81 NFP_IPSEC_HASH_SHA384_192, 82 NFP_IPSEC_HASH_SHA512_256, 83 NFP_IPSEC_HASH_GF128_128, 84 NFP_IPSEC_HASH_POLY1305_128 85 }; 86 87 /* IPSEC_CFG_MSSG_ADD_SA */ 88 struct nfp_ipsec_cfg_add_sa { 89 u32 ciph_key[8]; /* Cipher Key */ 90 union { 91 u32 auth_key[16]; /* Authentication Key */ 92 struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */ 93 u32 salt; /* Initialized with SA */ 94 u32 resv[15]; 95 } aesgcm_fields; 96 }; 97 struct sa_ctrl_word { 98 uint32_t hash :4; /* From nfp_ipsec_sa_hash_type */ 99 uint32_t cimode :4; /* From nfp_ipsec_sa_cipher_mode */ 100 uint32_t cipher :4; /* From nfp_ipsec_sa_cipher */ 101 uint32_t mode :2; /* From nfp_ipsec_sa_mode */ 102 uint32_t proto :2; /* From nfp_ipsec_sa_prot */ 103 uint32_t dir :1; /* SA direction */ 104 uint32_t resv0 :12; 105 uint32_t encap_dsbl:1; /* Encap/Decap disable */ 106 uint32_t resv1 :2; /* Must be set to 0 */ 107 } ctrl_word; 108 u32 spi; /* SPI Value */ 109 uint32_t pmtu_limit :16; /* PMTU Limit */ 110 uint32_t resv0 :5; 111 uint32_t ipv6 :1; /* Outbound IPv6 addr format */ 112 uint32_t resv1 :10; 113 u32 resv2[2]; 114 u32 src_ip[4]; /* Src IP addr */ 115 u32 dst_ip[4]; /* Dst IP addr */ 116 u32 resv3[6]; 117 }; 118 119 /* IPSEC_CFG_MSSG */ 120 struct nfp_ipsec_cfg_mssg { 121 union { 122 struct{ 123 uint32_t cmd:16; /* One of nfp_ipsec_cfg_mssg_cmd_codes */ 124 uint32_t rsp:16; /* One of nfp_ipsec_cfg_mssg_rsp_codes */ 125 uint32_t sa_idx:16; /* SA table index */ 126 uint32_t spare0:16; 127 struct nfp_ipsec_cfg_add_sa cfg_add_sa; 128 }; 129 u32 raw[64]; 130 }; 131 }; 132 133 static int nfp_ipsec_cfg_cmd_issue(struct nfp_net *nn, int type, int saidx, 134 struct nfp_ipsec_cfg_mssg *msg) 135 { 136 int i, msg_size, ret; 137 138 msg->cmd = type; 139 msg->sa_idx = saidx; 140 msg->rsp = 0; 141 msg_size = ARRAY_SIZE(msg->raw); 142 143 for (i = 0; i < msg_size; i++) 144 nn_writel(nn, NFP_NET_CFG_MBOX_VAL + 4 * i, msg->raw[i]); 145 146 ret = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_IPSEC); 147 if (ret < 0) 148 return ret; 149 150 /* For now we always read the whole message response back */ 151 for (i = 0; i < msg_size; i++) 152 msg->raw[i] = nn_readl(nn, NFP_NET_CFG_MBOX_VAL + 4 * i); 153 154 switch (msg->rsp) { 155 case NFP_IPSEC_CFG_MSSG_OK: 156 return 0; 157 case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD: 158 return -EINVAL; 159 case NFP_IPSEC_CFG_MSSG_SA_VALID: 160 return -EEXIST; 161 case NFP_IPSEC_CFG_MSSG_FAILED: 162 case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED: 163 case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED: 164 return -EIO; 165 default: 166 return -EINVAL; 167 } 168 } 169 170 static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen) 171 { 172 bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC); 173 174 switch (keylen) { 175 case 128: 176 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL : 177 NFP_IPSEC_CIPHER_AES128; 178 break; 179 case 192: 180 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL : 181 NFP_IPSEC_CIPHER_AES192; 182 break; 183 case 256: 184 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL : 185 NFP_IPSEC_CIPHER_AES256; 186 break; 187 default: 188 return -EINVAL; 189 } 190 191 return 0; 192 } 193 194 static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 195 { 196 switch (*trunc_len) { 197 case 96: 198 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96; 199 break; 200 case 128: 201 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128; 202 break; 203 default: 204 *trunc_len = 0; 205 } 206 } 207 208 static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 209 { 210 switch (*trunc_len) { 211 case 96: 212 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96; 213 break; 214 case 80: 215 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80; 216 break; 217 default: 218 *trunc_len = 0; 219 } 220 } 221 222 static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 223 { 224 switch (*trunc_len) { 225 case 96: 226 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96; 227 break; 228 case 128: 229 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128; 230 break; 231 default: 232 *trunc_len = 0; 233 } 234 } 235 236 static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 237 { 238 switch (*trunc_len) { 239 case 96: 240 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96; 241 break; 242 case 192: 243 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192; 244 break; 245 default: 246 *trunc_len = 0; 247 } 248 } 249 250 static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 251 { 252 switch (*trunc_len) { 253 case 96: 254 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96; 255 break; 256 case 256: 257 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256; 258 break; 259 default: 260 *trunc_len = 0; 261 } 262 } 263 264 static int nfp_net_xfrm_add_state(struct xfrm_state *x, 265 struct netlink_ext_ack *extack) 266 { 267 struct net_device *netdev = x->xso.dev; 268 struct nfp_ipsec_cfg_mssg msg = {}; 269 int i, key_len, trunc_len, err = 0; 270 struct nfp_ipsec_cfg_add_sa *cfg; 271 struct nfp_net *nn; 272 unsigned int saidx; 273 274 nn = netdev_priv(netdev); 275 cfg = &msg.cfg_add_sa; 276 277 /* General */ 278 switch (x->props.mode) { 279 case XFRM_MODE_TUNNEL: 280 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL; 281 break; 282 case XFRM_MODE_TRANSPORT: 283 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT; 284 break; 285 default: 286 NL_SET_ERR_MSG_MOD(extack, "Unsupported mode for xfrm offload"); 287 return -EINVAL; 288 } 289 290 switch (x->id.proto) { 291 case IPPROTO_ESP: 292 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP; 293 break; 294 case IPPROTO_AH: 295 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH; 296 break; 297 default: 298 NL_SET_ERR_MSG_MOD(extack, "Unsupported protocol for xfrm offload"); 299 return -EINVAL; 300 } 301 302 if (x->props.flags & XFRM_STATE_ESN) { 303 NL_SET_ERR_MSG_MOD(extack, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload"); 304 return -EINVAL; 305 } 306 307 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { 308 NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload type"); 309 return -EINVAL; 310 } 311 312 cfg->spi = ntohl(x->id.spi); 313 314 /* Hash/Authentication */ 315 if (x->aalg) 316 trunc_len = x->aalg->alg_trunc_len; 317 else 318 trunc_len = 0; 319 320 switch (x->props.aalgo) { 321 case SADB_AALG_NONE: 322 if (x->aead) { 323 trunc_len = -1; 324 } else { 325 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm"); 326 return -EINVAL; 327 } 328 break; 329 case SADB_X_AALG_NULL: 330 cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE; 331 trunc_len = -1; 332 break; 333 case SADB_AALG_MD5HMAC: 334 if (nn->pdev->device == PCI_DEVICE_ID_NFP3800) { 335 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm"); 336 return -EINVAL; 337 } 338 set_md5hmac(cfg, &trunc_len); 339 break; 340 case SADB_AALG_SHA1HMAC: 341 set_sha1hmac(cfg, &trunc_len); 342 break; 343 case SADB_X_AALG_SHA2_256HMAC: 344 set_sha2_256hmac(cfg, &trunc_len); 345 break; 346 case SADB_X_AALG_SHA2_384HMAC: 347 set_sha2_384hmac(cfg, &trunc_len); 348 break; 349 case SADB_X_AALG_SHA2_512HMAC: 350 set_sha2_512hmac(cfg, &trunc_len); 351 break; 352 default: 353 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm"); 354 return -EINVAL; 355 } 356 357 if (!trunc_len) { 358 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm trunc length"); 359 return -EINVAL; 360 } 361 362 if (x->aalg) { 363 key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE); 364 if (key_len > sizeof(cfg->auth_key)) { 365 NL_SET_ERR_MSG_MOD(extack, "Insufficient space for offloaded auth key"); 366 return -EINVAL; 367 } 368 for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++) 369 cfg->auth_key[i] = get_unaligned_be32(x->aalg->alg_key + 370 sizeof(cfg->auth_key[0]) * i); 371 } 372 373 /* Encryption */ 374 switch (x->props.ealgo) { 375 case SADB_EALG_NONE: 376 case SADB_EALG_NULL: 377 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 378 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL; 379 break; 380 case SADB_EALG_3DESCBC: 381 if (nn->pdev->device == PCI_DEVICE_ID_NFP3800) { 382 NL_SET_ERR_MSG_MOD(extack, "Unsupported encryption algorithm for offload"); 383 return -EINVAL; 384 } 385 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 386 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES; 387 break; 388 case SADB_X_EALG_AES_GCM_ICV16: 389 case SADB_X_EALG_NULL_AES_GMAC: 390 if (!x->aead) { 391 NL_SET_ERR_MSG_MOD(extack, "Invalid AES key data"); 392 return -EINVAL; 393 } 394 395 if (x->aead->alg_icv_len != 128) { 396 NL_SET_ERR_MSG_MOD(extack, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16"); 397 return -EINVAL; 398 } 399 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR; 400 cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128; 401 402 /* Aead->alg_key_len includes 32-bit salt */ 403 if (set_aes_keylen(cfg, x->props.ealgo, x->aead->alg_key_len - 32)) { 404 NL_SET_ERR_MSG_MOD(extack, "Unsupported AES key length"); 405 return -EINVAL; 406 } 407 break; 408 case SADB_X_EALG_AESCBC: 409 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 410 if (!x->ealg) { 411 NL_SET_ERR_MSG_MOD(extack, "Invalid AES key data"); 412 return -EINVAL; 413 } 414 if (set_aes_keylen(cfg, x->props.ealgo, x->ealg->alg_key_len) < 0) { 415 NL_SET_ERR_MSG_MOD(extack, "Unsupported AES key length"); 416 return -EINVAL; 417 } 418 break; 419 default: 420 NL_SET_ERR_MSG_MOD(extack, "Unsupported encryption algorithm for offload"); 421 return -EINVAL; 422 } 423 424 if (x->aead) { 425 int salt_len = 4; 426 427 key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE); 428 key_len -= salt_len; 429 430 if (key_len > sizeof(cfg->ciph_key)) { 431 NL_SET_ERR_MSG_MOD(extack, "aead: Insufficient space for offloaded key"); 432 return -EINVAL; 433 } 434 435 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 436 cfg->ciph_key[i] = get_unaligned_be32(x->aead->alg_key + 437 sizeof(cfg->ciph_key[0]) * i); 438 439 /* Load up the salt */ 440 cfg->aesgcm_fields.salt = get_unaligned_be32(x->aead->alg_key + key_len); 441 } 442 443 if (x->ealg) { 444 key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE); 445 446 if (key_len > sizeof(cfg->ciph_key)) { 447 NL_SET_ERR_MSG_MOD(extack, "ealg: Insufficient space for offloaded key"); 448 return -EINVAL; 449 } 450 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 451 cfg->ciph_key[i] = get_unaligned_be32(x->ealg->alg_key + 452 sizeof(cfg->ciph_key[0]) * i); 453 } 454 455 /* IP related info */ 456 switch (x->props.family) { 457 case AF_INET: 458 cfg->ipv6 = 0; 459 cfg->src_ip[0] = ntohl(x->props.saddr.a4); 460 cfg->dst_ip[0] = ntohl(x->id.daddr.a4); 461 break; 462 case AF_INET6: 463 cfg->ipv6 = 1; 464 for (i = 0; i < 4; i++) { 465 cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]); 466 cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]); 467 } 468 break; 469 default: 470 NL_SET_ERR_MSG_MOD(extack, "Unsupported address family"); 471 return -EINVAL; 472 } 473 474 /* Maximum nic IPsec code could handle. Other limits may apply. */ 475 cfg->pmtu_limit = 0xffff; 476 cfg->ctrl_word.encap_dsbl = 1; 477 478 /* SA direction */ 479 cfg->ctrl_word.dir = x->xso.dir; 480 481 /* Find unused SA data*/ 482 err = xa_alloc(&nn->xa_ipsec, &saidx, x, 483 XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL); 484 if (err < 0) { 485 NL_SET_ERR_MSG_MOD(extack, "Unable to get sa_data number for IPsec"); 486 return err; 487 } 488 489 /* Allocate saidx and commit the SA */ 490 err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_ADD_SA, saidx, &msg); 491 if (err) { 492 xa_erase(&nn->xa_ipsec, saidx); 493 NL_SET_ERR_MSG_MOD(extack, "Failed to issue IPsec command"); 494 return err; 495 } 496 497 /* 0 is invalid offload_handle for kernel */ 498 x->xso.offload_handle = saidx + 1; 499 return 0; 500 } 501 502 static void nfp_net_xfrm_del_state(struct xfrm_state *x) 503 { 504 struct net_device *netdev = x->xso.dev; 505 struct nfp_ipsec_cfg_mssg msg; 506 struct nfp_net *nn; 507 int err; 508 509 nn = netdev_priv(netdev); 510 err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_INV_SA, 511 x->xso.offload_handle - 1, &msg); 512 if (err) 513 nn_warn(nn, "Failed to invalidate SA in hardware\n"); 514 515 xa_erase(&nn->xa_ipsec, x->xso.offload_handle - 1); 516 } 517 518 static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 519 { 520 if (x->props.family == AF_INET) 521 /* Offload with IPv4 options is not supported yet */ 522 return ip_hdr(skb)->ihl == 5; 523 524 /* Offload with IPv6 extension headers is not support yet */ 525 return !(ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)); 526 } 527 528 static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = { 529 .xdo_dev_state_add = nfp_net_xfrm_add_state, 530 .xdo_dev_state_delete = nfp_net_xfrm_del_state, 531 .xdo_dev_offload_ok = nfp_net_ipsec_offload_ok, 532 }; 533 534 void nfp_net_ipsec_init(struct nfp_net *nn) 535 { 536 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 537 return; 538 539 xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC); 540 nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops; 541 } 542 543 void nfp_net_ipsec_clean(struct nfp_net *nn) 544 { 545 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 546 return; 547 548 WARN_ON(!xa_empty(&nn->xa_ipsec)); 549 xa_destroy(&nn->xa_ipsec); 550 } 551 552 bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb, 553 struct nfp_ipsec_offload *offload_info) 554 { 555 struct xfrm_offload *xo = xfrm_offload(skb); 556 struct xfrm_state *x; 557 558 x = xfrm_input_state(skb); 559 if (!x) 560 return false; 561 562 offload_info->seq_hi = xo->seq.hi; 563 offload_info->seq_low = xo->seq.low; 564 offload_info->handle = x->xso.offload_handle; 565 566 return true; 567 } 568 569 int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb) 570 { 571 struct net_device *netdev = skb->dev; 572 struct xfrm_offload *xo; 573 struct xfrm_state *x; 574 struct sec_path *sp; 575 struct nfp_net *nn; 576 u32 saidx; 577 578 nn = netdev_priv(netdev); 579 580 saidx = meta->ipsec_saidx - 1; 581 if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT) 582 return -EINVAL; 583 584 sp = secpath_set(skb); 585 if (unlikely(!sp)) 586 return -ENOMEM; 587 588 xa_lock(&nn->xa_ipsec); 589 x = xa_load(&nn->xa_ipsec, saidx); 590 xa_unlock(&nn->xa_ipsec); 591 if (!x) 592 return -EINVAL; 593 594 xfrm_state_hold(x); 595 sp->xvec[sp->len++] = x; 596 sp->olen++; 597 xo = xfrm_offload(skb); 598 xo->flags = CRYPTO_DONE; 599 xo->status = CRYPTO_SUCCESS; 600 601 return 0; 602 } 603