1 /* 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <crypto/internal/geniv.h> 35 #include <crypto/aead.h> 36 #include <linux/inetdevice.h> 37 #include <linux/netdevice.h> 38 #include <linux/module.h> 39 40 #include "en.h" 41 #include "en_accel/ipsec.h" 42 #include "en_accel/ipsec_rxtx.h" 43 #include "en_accel/ipsec_fs.h" 44 45 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) 46 { 47 struct mlx5e_ipsec_sa_entry *sa; 48 49 if (!x) 50 return NULL; 51 52 sa = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; 53 if (!sa) 54 return NULL; 55 56 WARN_ON(sa->x != x); 57 return sa; 58 } 59 60 struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, 61 unsigned int handle) 62 { 63 struct mlx5e_ipsec_sa_entry *sa_entry; 64 struct xfrm_state *ret = NULL; 65 66 rcu_read_lock(); 67 hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle) 68 if (sa_entry->handle == handle) { 69 ret = sa_entry->x; 70 xfrm_state_hold(ret); 71 break; 72 } 73 rcu_read_unlock(); 74 75 return ret; 76 } 77 78 static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry, 79 unsigned int handle) 80 { 81 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 82 struct mlx5e_ipsec_sa_entry *_sa_entry; 83 unsigned long flags; 84 85 rcu_read_lock(); 86 hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle) 87 if (_sa_entry->handle == handle) { 88 rcu_read_unlock(); 89 return -EEXIST; 90 } 91 rcu_read_unlock(); 92 93 spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); 94 sa_entry->handle = handle; 95 hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle); 96 spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); 97 98 return 0; 99 } 100 101 static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) 102 { 103 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 104 unsigned long flags; 105 106 spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); 107 hash_del_rcu(&sa_entry->hlist); 108 spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); 109 } 110 111 static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) 112 { 113 struct xfrm_replay_state_esn *replay_esn; 114 u32 seq_bottom = 0; 115 u8 overlap; 116 u32 *esn; 117 118 if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) { 119 sa_entry->esn_state.trigger = 0; 120 return false; 121 } 122 123 replay_esn = sa_entry->x->replay_esn; 124 if (replay_esn->seq >= replay_esn->replay_window) 125 seq_bottom = replay_esn->seq - replay_esn->replay_window + 1; 126 127 overlap = sa_entry->esn_state.overlap; 128 129 sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x, 130 htonl(seq_bottom)); 131 esn = &sa_entry->esn_state.esn; 132 133 sa_entry->esn_state.trigger = 1; 134 if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) { 135 ++(*esn); 136 sa_entry->esn_state.overlap = 0; 137 return true; 138 } else if (unlikely(!overlap && 139 (seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) { 140 sa_entry->esn_state.overlap = 1; 141 return true; 142 } 143 144 return false; 145 } 146 147 static void 148 mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 149 struct mlx5_accel_esp_xfrm_attrs *attrs) 150 { 151 struct xfrm_state *x = sa_entry->x; 152 struct aes_gcm_keymat *aes_gcm = &attrs->keymat.aes_gcm; 153 struct aead_geniv_ctx *geniv_ctx; 154 struct crypto_aead *aead; 155 unsigned int crypto_data_len, key_len; 156 int ivsize; 157 158 memset(attrs, 0, sizeof(*attrs)); 159 160 /* key */ 161 crypto_data_len = (x->aead->alg_key_len + 7) / 8; 162 key_len = crypto_data_len - 4; /* 4 bytes salt at end */ 163 164 memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len); 165 aes_gcm->key_len = key_len * 8; 166 167 /* salt and seq_iv */ 168 aead = x->data; 169 geniv_ctx = crypto_aead_ctx(aead); 170 ivsize = crypto_aead_ivsize(aead); 171 memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize); 172 memcpy(&aes_gcm->salt, x->aead->alg_key + key_len, 173 sizeof(aes_gcm->salt)); 174 175 /* iv len */ 176 aes_gcm->icv_len = x->aead->alg_icv_len; 177 178 /* esn */ 179 if (sa_entry->esn_state.trigger) { 180 attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; 181 attrs->esn = sa_entry->esn_state.esn; 182 if (sa_entry->esn_state.overlap) 183 attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 184 } 185 186 /* rx handle */ 187 attrs->sa_handle = sa_entry->handle; 188 189 /* algo type */ 190 attrs->keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM; 191 192 /* action */ 193 attrs->action = (!(x->xso.flags & XFRM_OFFLOAD_INBOUND)) ? 194 MLX5_ACCEL_ESP_ACTION_ENCRYPT : 195 MLX5_ACCEL_ESP_ACTION_DECRYPT; 196 /* flags */ 197 attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ? 198 MLX5_ACCEL_ESP_FLAGS_TRANSPORT : 199 MLX5_ACCEL_ESP_FLAGS_TUNNEL; 200 201 /* spi */ 202 attrs->spi = x->id.spi; 203 204 /* source , destination ips */ 205 memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); 206 memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); 207 attrs->is_ipv6 = (x->props.family != AF_INET); 208 } 209 210 static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) 211 { 212 struct net_device *netdev = x->xso.real_dev; 213 struct mlx5e_priv *priv; 214 215 priv = netdev_priv(netdev); 216 217 if (x->props.aalgo != SADB_AALG_NONE) { 218 netdev_info(netdev, "Cannot offload authenticated xfrm states\n"); 219 return -EINVAL; 220 } 221 if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) { 222 netdev_info(netdev, "Only AES-GCM-ICV16 xfrm state may be offloaded\n"); 223 return -EINVAL; 224 } 225 if (x->props.calgo != SADB_X_CALG_NONE) { 226 netdev_info(netdev, "Cannot offload compressed xfrm states\n"); 227 return -EINVAL; 228 } 229 if (x->props.flags & XFRM_STATE_ESN && 230 !(mlx5_accel_ipsec_device_caps(priv->mdev) & 231 MLX5_ACCEL_IPSEC_CAP_ESN)) { 232 netdev_info(netdev, "Cannot offload ESN xfrm states\n"); 233 return -EINVAL; 234 } 235 if (x->props.family != AF_INET && 236 x->props.family != AF_INET6) { 237 netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n"); 238 return -EINVAL; 239 } 240 if (x->props.mode != XFRM_MODE_TRANSPORT && 241 x->props.mode != XFRM_MODE_TUNNEL) { 242 dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n"); 243 return -EINVAL; 244 } 245 if (x->id.proto != IPPROTO_ESP) { 246 netdev_info(netdev, "Only ESP xfrm state may be offloaded\n"); 247 return -EINVAL; 248 } 249 if (x->encap) { 250 netdev_info(netdev, "Encapsulated xfrm state may not be offloaded\n"); 251 return -EINVAL; 252 } 253 if (!x->aead) { 254 netdev_info(netdev, "Cannot offload xfrm states without aead\n"); 255 return -EINVAL; 256 } 257 if (x->aead->alg_icv_len != 128) { 258 netdev_info(netdev, "Cannot offload xfrm states with AEAD ICV length other than 128bit\n"); 259 return -EINVAL; 260 } 261 if ((x->aead->alg_key_len != 128 + 32) && 262 (x->aead->alg_key_len != 256 + 32)) { 263 netdev_info(netdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); 264 return -EINVAL; 265 } 266 if (x->tfcpad) { 267 netdev_info(netdev, "Cannot offload xfrm states with tfc padding\n"); 268 return -EINVAL; 269 } 270 if (!x->geniv) { 271 netdev_info(netdev, "Cannot offload xfrm states without geniv\n"); 272 return -EINVAL; 273 } 274 if (strcmp(x->geniv, "seqiv")) { 275 netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n"); 276 return -EINVAL; 277 } 278 if (x->props.family == AF_INET6 && 279 !(mlx5_accel_ipsec_device_caps(priv->mdev) & 280 MLX5_ACCEL_IPSEC_CAP_IPV6)) { 281 netdev_info(netdev, "IPv6 xfrm state offload is not supported by this device\n"); 282 return -EINVAL; 283 } 284 return 0; 285 } 286 287 static int mlx5e_xfrm_fs_add_rule(struct mlx5e_priv *priv, 288 struct mlx5e_ipsec_sa_entry *sa_entry) 289 { 290 if (!mlx5_is_ipsec_device(priv->mdev)) 291 return 0; 292 293 return mlx5e_accel_ipsec_fs_add_rule(priv, &sa_entry->xfrm->attrs, 294 sa_entry->ipsec_obj_id, 295 &sa_entry->ipsec_rule); 296 } 297 298 static void mlx5e_xfrm_fs_del_rule(struct mlx5e_priv *priv, 299 struct mlx5e_ipsec_sa_entry *sa_entry) 300 { 301 if (!mlx5_is_ipsec_device(priv->mdev)) 302 return; 303 304 mlx5e_accel_ipsec_fs_del_rule(priv, &sa_entry->xfrm->attrs, 305 &sa_entry->ipsec_rule); 306 } 307 308 static int mlx5e_xfrm_add_state(struct xfrm_state *x) 309 { 310 struct mlx5e_ipsec_sa_entry *sa_entry = NULL; 311 struct net_device *netdev = x->xso.real_dev; 312 struct mlx5_accel_esp_xfrm_attrs attrs; 313 struct mlx5e_priv *priv; 314 unsigned int sa_handle; 315 int err; 316 317 priv = netdev_priv(netdev); 318 319 err = mlx5e_xfrm_validate_state(x); 320 if (err) 321 return err; 322 323 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); 324 if (!sa_entry) { 325 err = -ENOMEM; 326 goto out; 327 } 328 329 sa_entry->x = x; 330 sa_entry->ipsec = priv->ipsec; 331 332 /* check esn */ 333 mlx5e_ipsec_update_esn_state(sa_entry); 334 335 /* create xfrm */ 336 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs); 337 sa_entry->xfrm = 338 mlx5_accel_esp_create_xfrm(priv->mdev, &attrs, 339 MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA); 340 if (IS_ERR(sa_entry->xfrm)) { 341 err = PTR_ERR(sa_entry->xfrm); 342 goto err_sa_entry; 343 } 344 345 /* create hw context */ 346 sa_entry->hw_context = 347 mlx5_accel_esp_create_hw_context(priv->mdev, 348 sa_entry->xfrm, 349 &sa_handle); 350 if (IS_ERR(sa_entry->hw_context)) { 351 err = PTR_ERR(sa_entry->hw_context); 352 goto err_xfrm; 353 } 354 355 sa_entry->ipsec_obj_id = sa_handle; 356 err = mlx5e_xfrm_fs_add_rule(priv, sa_entry); 357 if (err) 358 goto err_hw_ctx; 359 360 if (x->xso.flags & XFRM_OFFLOAD_INBOUND) { 361 err = mlx5e_ipsec_sadb_rx_add(sa_entry, sa_handle); 362 if (err) 363 goto err_add_rule; 364 } else { 365 sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? 366 mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; 367 } 368 369 x->xso.offload_handle = (unsigned long)sa_entry; 370 goto out; 371 372 err_add_rule: 373 mlx5e_xfrm_fs_del_rule(priv, sa_entry); 374 err_hw_ctx: 375 mlx5_accel_esp_free_hw_context(priv->mdev, sa_entry->hw_context); 376 err_xfrm: 377 mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm); 378 err_sa_entry: 379 kfree(sa_entry); 380 381 out: 382 return err; 383 } 384 385 static void mlx5e_xfrm_del_state(struct xfrm_state *x) 386 { 387 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 388 389 if (!sa_entry) 390 return; 391 392 if (x->xso.flags & XFRM_OFFLOAD_INBOUND) 393 mlx5e_ipsec_sadb_rx_del(sa_entry); 394 } 395 396 static void mlx5e_xfrm_free_state(struct xfrm_state *x) 397 { 398 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 399 struct mlx5e_priv *priv = netdev_priv(x->xso.dev); 400 401 if (!sa_entry) 402 return; 403 404 if (sa_entry->hw_context) { 405 flush_workqueue(sa_entry->ipsec->wq); 406 mlx5e_xfrm_fs_del_rule(priv, sa_entry); 407 mlx5_accel_esp_free_hw_context(sa_entry->xfrm->mdev, sa_entry->hw_context); 408 mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm); 409 } 410 411 kfree(sa_entry); 412 } 413 414 int mlx5e_ipsec_init(struct mlx5e_priv *priv) 415 { 416 struct mlx5e_ipsec *ipsec = NULL; 417 418 if (!MLX5_IPSEC_DEV(priv->mdev)) { 419 netdev_dbg(priv->netdev, "Not an IPSec offload device\n"); 420 return 0; 421 } 422 423 ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL); 424 if (!ipsec) 425 return -ENOMEM; 426 427 hash_init(ipsec->sadb_rx); 428 spin_lock_init(&ipsec->sadb_rx_lock); 429 ida_init(&ipsec->halloc); 430 ipsec->en_priv = priv; 431 ipsec->no_trailer = !!(mlx5_accel_ipsec_device_caps(priv->mdev) & 432 MLX5_ACCEL_IPSEC_CAP_RX_NO_TRAILER); 433 ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, 434 priv->netdev->name); 435 if (!ipsec->wq) { 436 kfree(ipsec); 437 return -ENOMEM; 438 } 439 440 priv->ipsec = ipsec; 441 mlx5e_accel_ipsec_fs_init(priv); 442 netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); 443 return 0; 444 } 445 446 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 447 { 448 struct mlx5e_ipsec *ipsec = priv->ipsec; 449 450 if (!ipsec) 451 return; 452 453 mlx5e_accel_ipsec_fs_cleanup(priv); 454 destroy_workqueue(ipsec->wq); 455 456 ida_destroy(&ipsec->halloc); 457 kfree(ipsec); 458 priv->ipsec = NULL; 459 } 460 461 static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 462 { 463 if (x->props.family == AF_INET) { 464 /* Offload with IPv4 options is not supported yet */ 465 if (ip_hdr(skb)->ihl > 5) 466 return false; 467 } else { 468 /* Offload with IPv6 extension headers is not support yet */ 469 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)) 470 return false; 471 } 472 473 return true; 474 } 475 476 struct mlx5e_ipsec_modify_state_work { 477 struct work_struct work; 478 struct mlx5_accel_esp_xfrm_attrs attrs; 479 struct mlx5e_ipsec_sa_entry *sa_entry; 480 }; 481 482 static void _update_xfrm_state(struct work_struct *work) 483 { 484 int ret; 485 struct mlx5e_ipsec_modify_state_work *modify_work = 486 container_of(work, struct mlx5e_ipsec_modify_state_work, work); 487 struct mlx5e_ipsec_sa_entry *sa_entry = modify_work->sa_entry; 488 489 ret = mlx5_accel_esp_modify_xfrm(sa_entry->xfrm, 490 &modify_work->attrs); 491 if (ret) 492 netdev_warn(sa_entry->ipsec->en_priv->netdev, 493 "Not an IPSec offload device\n"); 494 495 kfree(modify_work); 496 } 497 498 static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) 499 { 500 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 501 struct mlx5e_ipsec_modify_state_work *modify_work; 502 bool need_update; 503 504 if (!sa_entry) 505 return; 506 507 need_update = mlx5e_ipsec_update_esn_state(sa_entry); 508 if (!need_update) 509 return; 510 511 modify_work = kzalloc(sizeof(*modify_work), GFP_ATOMIC); 512 if (!modify_work) 513 return; 514 515 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs); 516 modify_work->sa_entry = sa_entry; 517 518 INIT_WORK(&modify_work->work, _update_xfrm_state); 519 WARN_ON(!queue_work(sa_entry->ipsec->wq, &modify_work->work)); 520 } 521 522 static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { 523 .xdo_dev_state_add = mlx5e_xfrm_add_state, 524 .xdo_dev_state_delete = mlx5e_xfrm_del_state, 525 .xdo_dev_state_free = mlx5e_xfrm_free_state, 526 .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, 527 .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, 528 }; 529 530 void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 531 { 532 struct mlx5_core_dev *mdev = priv->mdev; 533 struct net_device *netdev = priv->netdev; 534 535 if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_ESP) || 536 !MLX5_CAP_ETH(mdev, swp)) { 537 mlx5_core_dbg(mdev, "mlx5e: ESP and SWP offload not supported\n"); 538 return; 539 } 540 541 mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n"); 542 netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; 543 netdev->features |= NETIF_F_HW_ESP; 544 netdev->hw_enc_features |= NETIF_F_HW_ESP; 545 546 if (!MLX5_CAP_ETH(mdev, swp_csum)) { 547 mlx5_core_dbg(mdev, "mlx5e: SWP checksum not supported\n"); 548 return; 549 } 550 551 netdev->features |= NETIF_F_HW_ESP_TX_CSUM; 552 netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM; 553 554 if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_LSO) || 555 !MLX5_CAP_ETH(mdev, swp_lso)) { 556 mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n"); 557 return; 558 } 559 560 if (mlx5_is_ipsec_device(mdev)) 561 netdev->gso_partial_features |= NETIF_F_GSO_ESP; 562 563 mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n"); 564 netdev->features |= NETIF_F_GSO_ESP; 565 netdev->hw_features |= NETIF_F_GSO_ESP; 566 netdev->hw_enc_features |= NETIF_F_GSO_ESP; 567 } 568