1 /* 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <crypto/internal/geniv.h> 35 #include <crypto/aead.h> 36 #include <linux/inetdevice.h> 37 #include <linux/netdevice.h> 38 39 #include "en.h" 40 #include "ipsec.h" 41 #include "ipsec_rxtx.h" 42 43 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) 44 { 45 return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; 46 } 47 48 struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, 49 unsigned int handle) 50 { 51 struct mlx5e_ipsec_sa_entry *sa_entry; 52 struct xfrm_state *ret = NULL; 53 54 rcu_read_lock(); 55 hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle) 56 if (sa_entry->handle == handle) { 57 ret = sa_entry->x; 58 xfrm_state_hold(ret); 59 break; 60 } 61 rcu_read_unlock(); 62 63 return ret; 64 } 65 66 static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry) 67 { 68 unsigned int handle = sa_entry->ipsec_obj_id; 69 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 70 struct mlx5e_ipsec_sa_entry *_sa_entry; 71 unsigned long flags; 72 73 rcu_read_lock(); 74 hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle) 75 if (_sa_entry->handle == handle) { 76 rcu_read_unlock(); 77 return -EEXIST; 78 } 79 rcu_read_unlock(); 80 81 spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); 82 sa_entry->handle = handle; 83 hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle); 84 spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); 85 86 return 0; 87 } 88 89 static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) 90 { 91 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 92 unsigned long flags; 93 94 spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); 95 hash_del_rcu(&sa_entry->hlist); 96 spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); 97 } 98 99 static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) 100 { 101 struct xfrm_replay_state_esn *replay_esn; 102 u32 seq_bottom = 0; 103 u8 overlap; 104 u32 *esn; 105 106 if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) { 107 sa_entry->esn_state.trigger = 0; 108 return false; 109 } 110 111 replay_esn = sa_entry->x->replay_esn; 112 if (replay_esn->seq >= replay_esn->replay_window) 113 seq_bottom = replay_esn->seq - replay_esn->replay_window + 1; 114 115 overlap = sa_entry->esn_state.overlap; 116 117 sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x, 118 htonl(seq_bottom)); 119 esn = &sa_entry->esn_state.esn; 120 121 sa_entry->esn_state.trigger = 1; 122 if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) { 123 ++(*esn); 124 sa_entry->esn_state.overlap = 0; 125 return true; 126 } else if (unlikely(!overlap && 127 (seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) { 128 sa_entry->esn_state.overlap = 1; 129 return true; 130 } 131 132 return false; 133 } 134 135 static void 136 mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 137 struct mlx5_accel_esp_xfrm_attrs *attrs) 138 { 139 struct xfrm_state *x = sa_entry->x; 140 struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; 141 struct aead_geniv_ctx *geniv_ctx; 142 struct crypto_aead *aead; 143 unsigned int crypto_data_len, key_len; 144 int ivsize; 145 146 memset(attrs, 0, sizeof(*attrs)); 147 148 /* key */ 149 crypto_data_len = (x->aead->alg_key_len + 7) / 8; 150 key_len = crypto_data_len - 4; /* 4 bytes salt at end */ 151 152 memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len); 153 aes_gcm->key_len = key_len * 8; 154 155 /* salt and seq_iv */ 156 aead = x->data; 157 geniv_ctx = crypto_aead_ctx(aead); 158 ivsize = crypto_aead_ivsize(aead); 159 memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize); 160 memcpy(&aes_gcm->salt, x->aead->alg_key + key_len, 161 sizeof(aes_gcm->salt)); 162 163 /* iv len */ 164 aes_gcm->icv_len = x->aead->alg_icv_len; 165 166 /* esn */ 167 if (sa_entry->esn_state.trigger) { 168 attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; 169 attrs->esn = sa_entry->esn_state.esn; 170 if (sa_entry->esn_state.overlap) 171 attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 172 } 173 174 /* action */ 175 attrs->action = (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) ? 176 MLX5_ACCEL_ESP_ACTION_ENCRYPT : 177 MLX5_ACCEL_ESP_ACTION_DECRYPT; 178 /* flags */ 179 attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ? 180 MLX5_ACCEL_ESP_FLAGS_TRANSPORT : 181 MLX5_ACCEL_ESP_FLAGS_TUNNEL; 182 183 /* spi */ 184 attrs->spi = be32_to_cpu(x->id.spi); 185 186 /* source , destination ips */ 187 memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); 188 memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); 189 attrs->is_ipv6 = (x->props.family != AF_INET); 190 } 191 192 static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) 193 { 194 struct net_device *netdev = x->xso.real_dev; 195 struct mlx5e_priv *priv; 196 197 priv = netdev_priv(netdev); 198 199 if (x->props.aalgo != SADB_AALG_NONE) { 200 netdev_info(netdev, "Cannot offload authenticated xfrm states\n"); 201 return -EINVAL; 202 } 203 if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) { 204 netdev_info(netdev, "Only AES-GCM-ICV16 xfrm state may be offloaded\n"); 205 return -EINVAL; 206 } 207 if (x->props.calgo != SADB_X_CALG_NONE) { 208 netdev_info(netdev, "Cannot offload compressed xfrm states\n"); 209 return -EINVAL; 210 } 211 if (x->props.flags & XFRM_STATE_ESN && 212 !(mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_ESN)) { 213 netdev_info(netdev, "Cannot offload ESN xfrm states\n"); 214 return -EINVAL; 215 } 216 if (x->props.family != AF_INET && 217 x->props.family != AF_INET6) { 218 netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n"); 219 return -EINVAL; 220 } 221 if (x->props.mode != XFRM_MODE_TRANSPORT && 222 x->props.mode != XFRM_MODE_TUNNEL) { 223 dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n"); 224 return -EINVAL; 225 } 226 if (x->id.proto != IPPROTO_ESP) { 227 netdev_info(netdev, "Only ESP xfrm state may be offloaded\n"); 228 return -EINVAL; 229 } 230 if (x->encap) { 231 netdev_info(netdev, "Encapsulated xfrm state may not be offloaded\n"); 232 return -EINVAL; 233 } 234 if (!x->aead) { 235 netdev_info(netdev, "Cannot offload xfrm states without aead\n"); 236 return -EINVAL; 237 } 238 if (x->aead->alg_icv_len != 128) { 239 netdev_info(netdev, "Cannot offload xfrm states with AEAD ICV length other than 128bit\n"); 240 return -EINVAL; 241 } 242 if ((x->aead->alg_key_len != 128 + 32) && 243 (x->aead->alg_key_len != 256 + 32)) { 244 netdev_info(netdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); 245 return -EINVAL; 246 } 247 if (x->tfcpad) { 248 netdev_info(netdev, "Cannot offload xfrm states with tfc padding\n"); 249 return -EINVAL; 250 } 251 if (!x->geniv) { 252 netdev_info(netdev, "Cannot offload xfrm states without geniv\n"); 253 return -EINVAL; 254 } 255 if (strcmp(x->geniv, "seqiv")) { 256 netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n"); 257 return -EINVAL; 258 } 259 return 0; 260 } 261 262 static void _update_xfrm_state(struct work_struct *work) 263 { 264 struct mlx5e_ipsec_modify_state_work *modify_work = 265 container_of(work, struct mlx5e_ipsec_modify_state_work, work); 266 struct mlx5e_ipsec_sa_entry *sa_entry = container_of( 267 modify_work, struct mlx5e_ipsec_sa_entry, modify_work); 268 269 mlx5_accel_esp_modify_xfrm(sa_entry, &modify_work->attrs); 270 } 271 272 static int mlx5e_xfrm_add_state(struct xfrm_state *x) 273 { 274 struct mlx5e_ipsec_sa_entry *sa_entry = NULL; 275 struct net_device *netdev = x->xso.real_dev; 276 struct mlx5e_priv *priv; 277 int err; 278 279 priv = netdev_priv(netdev); 280 if (!priv->ipsec) 281 return -EOPNOTSUPP; 282 283 err = mlx5e_xfrm_validate_state(x); 284 if (err) 285 return err; 286 287 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); 288 if (!sa_entry) { 289 err = -ENOMEM; 290 goto out; 291 } 292 293 sa_entry->x = x; 294 sa_entry->ipsec = priv->ipsec; 295 296 /* check esn */ 297 mlx5e_ipsec_update_esn_state(sa_entry); 298 299 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs); 300 /* create hw context */ 301 err = mlx5_ipsec_create_sa_ctx(sa_entry); 302 if (err) 303 goto err_xfrm; 304 305 err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry); 306 if (err) 307 goto err_hw_ctx; 308 309 if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) { 310 err = mlx5e_ipsec_sadb_rx_add(sa_entry); 311 if (err) 312 goto err_add_rule; 313 } else { 314 sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? 315 mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; 316 } 317 318 INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state); 319 x->xso.offload_handle = (unsigned long)sa_entry; 320 goto out; 321 322 err_add_rule: 323 mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); 324 err_hw_ctx: 325 mlx5_ipsec_free_sa_ctx(sa_entry); 326 err_xfrm: 327 kfree(sa_entry); 328 out: 329 return err; 330 } 331 332 static void mlx5e_xfrm_del_state(struct xfrm_state *x) 333 { 334 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 335 336 if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) 337 mlx5e_ipsec_sadb_rx_del(sa_entry); 338 } 339 340 static void mlx5e_xfrm_free_state(struct xfrm_state *x) 341 { 342 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 343 struct mlx5e_priv *priv = netdev_priv(x->xso.dev); 344 345 cancel_work_sync(&sa_entry->modify_work.work); 346 mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); 347 mlx5_ipsec_free_sa_ctx(sa_entry); 348 kfree(sa_entry); 349 } 350 351 int mlx5e_ipsec_init(struct mlx5e_priv *priv) 352 { 353 struct mlx5e_ipsec *ipsec; 354 int ret; 355 356 if (!mlx5_ipsec_device_caps(priv->mdev)) { 357 netdev_dbg(priv->netdev, "Not an IPSec offload device\n"); 358 return 0; 359 } 360 361 ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL); 362 if (!ipsec) 363 return -ENOMEM; 364 365 hash_init(ipsec->sadb_rx); 366 spin_lock_init(&ipsec->sadb_rx_lock); 367 ipsec->mdev = priv->mdev; 368 ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, 369 priv->netdev->name); 370 if (!ipsec->wq) { 371 ret = -ENOMEM; 372 goto err_wq; 373 } 374 375 ret = mlx5e_accel_ipsec_fs_init(ipsec); 376 if (ret) 377 goto err_fs_init; 378 379 priv->ipsec = ipsec; 380 netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); 381 return 0; 382 383 err_fs_init: 384 destroy_workqueue(ipsec->wq); 385 err_wq: 386 kfree(ipsec); 387 return (ret != -EOPNOTSUPP) ? ret : 0; 388 } 389 390 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 391 { 392 struct mlx5e_ipsec *ipsec = priv->ipsec; 393 394 if (!ipsec) 395 return; 396 397 mlx5e_accel_ipsec_fs_cleanup(ipsec); 398 destroy_workqueue(ipsec->wq); 399 kfree(ipsec); 400 priv->ipsec = NULL; 401 } 402 403 static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 404 { 405 if (x->props.family == AF_INET) { 406 /* Offload with IPv4 options is not supported yet */ 407 if (ip_hdr(skb)->ihl > 5) 408 return false; 409 } else { 410 /* Offload with IPv6 extension headers is not support yet */ 411 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)) 412 return false; 413 } 414 415 return true; 416 } 417 418 static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) 419 { 420 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 421 struct mlx5e_ipsec_modify_state_work *modify_work = 422 &sa_entry->modify_work; 423 bool need_update; 424 425 need_update = mlx5e_ipsec_update_esn_state(sa_entry); 426 if (!need_update) 427 return; 428 429 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs); 430 queue_work(sa_entry->ipsec->wq, &modify_work->work); 431 } 432 433 static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { 434 .xdo_dev_state_add = mlx5e_xfrm_add_state, 435 .xdo_dev_state_delete = mlx5e_xfrm_del_state, 436 .xdo_dev_state_free = mlx5e_xfrm_free_state, 437 .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, 438 .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, 439 }; 440 441 void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 442 { 443 struct mlx5_core_dev *mdev = priv->mdev; 444 struct net_device *netdev = priv->netdev; 445 446 if (!mlx5_ipsec_device_caps(mdev)) 447 return; 448 449 mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n"); 450 netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; 451 netdev->features |= NETIF_F_HW_ESP; 452 netdev->hw_enc_features |= NETIF_F_HW_ESP; 453 454 if (!MLX5_CAP_ETH(mdev, swp_csum)) { 455 mlx5_core_dbg(mdev, "mlx5e: SWP checksum not supported\n"); 456 return; 457 } 458 459 netdev->features |= NETIF_F_HW_ESP_TX_CSUM; 460 netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM; 461 462 if (!MLX5_CAP_ETH(mdev, swp_lso)) { 463 mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n"); 464 return; 465 } 466 467 netdev->gso_partial_features |= NETIF_F_GSO_ESP; 468 mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n"); 469 netdev->features |= NETIF_F_GSO_ESP; 470 netdev->hw_features |= NETIF_F_GSO_ESP; 471 netdev->hw_enc_features |= NETIF_F_GSO_ESP; 472 } 473