1 /* 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <crypto/internal/geniv.h> 35 #include <crypto/aead.h> 36 #include <linux/inetdevice.h> 37 #include <linux/netdevice.h> 38 39 #include "en.h" 40 #include "ipsec.h" 41 #include "ipsec_rxtx.h" 42 43 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) 44 { 45 return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; 46 } 47 48 struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, 49 unsigned int handle) 50 { 51 struct mlx5e_ipsec_sa_entry *sa_entry; 52 struct xfrm_state *ret = NULL; 53 54 rcu_read_lock(); 55 hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle) 56 if (sa_entry->handle == handle) { 57 ret = sa_entry->x; 58 xfrm_state_hold(ret); 59 break; 60 } 61 rcu_read_unlock(); 62 63 return ret; 64 } 65 66 static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry) 67 { 68 unsigned int handle = sa_entry->ipsec_obj_id; 69 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 70 struct mlx5e_ipsec_sa_entry *_sa_entry; 71 unsigned long flags; 72 73 rcu_read_lock(); 74 hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle) 75 if (_sa_entry->handle == handle) { 76 rcu_read_unlock(); 77 return -EEXIST; 78 } 79 rcu_read_unlock(); 80 81 spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); 82 sa_entry->handle = handle; 83 hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle); 84 spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); 85 86 return 0; 87 } 88 89 static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) 90 { 91 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 92 unsigned long flags; 93 94 spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); 95 hash_del_rcu(&sa_entry->hlist); 96 spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); 97 } 98 99 static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) 100 { 101 struct xfrm_replay_state_esn *replay_esn; 102 u32 seq_bottom = 0; 103 u8 overlap; 104 105 if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) { 106 sa_entry->esn_state.trigger = 0; 107 return false; 108 } 109 110 replay_esn = sa_entry->x->replay_esn; 111 if (replay_esn->seq >= replay_esn->replay_window) 112 seq_bottom = replay_esn->seq - replay_esn->replay_window + 1; 113 114 overlap = sa_entry->esn_state.overlap; 115 116 sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x, 117 htonl(seq_bottom)); 118 119 sa_entry->esn_state.trigger = 1; 120 if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) { 121 sa_entry->esn_state.overlap = 0; 122 return true; 123 } else if (unlikely(!overlap && 124 (seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) { 125 sa_entry->esn_state.overlap = 1; 126 return true; 127 } 128 129 return false; 130 } 131 132 static void 133 mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 134 struct mlx5_accel_esp_xfrm_attrs *attrs) 135 { 136 struct xfrm_state *x = sa_entry->x; 137 struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; 138 struct aead_geniv_ctx *geniv_ctx; 139 struct crypto_aead *aead; 140 unsigned int crypto_data_len, key_len; 141 int ivsize; 142 143 memset(attrs, 0, sizeof(*attrs)); 144 145 /* key */ 146 crypto_data_len = (x->aead->alg_key_len + 7) / 8; 147 key_len = crypto_data_len - 4; /* 4 bytes salt at end */ 148 149 memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len); 150 aes_gcm->key_len = key_len * 8; 151 152 /* salt and seq_iv */ 153 aead = x->data; 154 geniv_ctx = crypto_aead_ctx(aead); 155 ivsize = crypto_aead_ivsize(aead); 156 memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize); 157 memcpy(&aes_gcm->salt, x->aead->alg_key + key_len, 158 sizeof(aes_gcm->salt)); 159 160 /* iv len */ 161 aes_gcm->icv_len = x->aead->alg_icv_len; 162 163 /* esn */ 164 if (sa_entry->esn_state.trigger) { 165 attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; 166 attrs->esn = sa_entry->esn_state.esn; 167 if (sa_entry->esn_state.overlap) 168 attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 169 } 170 171 /* action */ 172 attrs->action = (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) ? 173 MLX5_ACCEL_ESP_ACTION_ENCRYPT : 174 MLX5_ACCEL_ESP_ACTION_DECRYPT; 175 /* flags */ 176 attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ? 177 MLX5_ACCEL_ESP_FLAGS_TRANSPORT : 178 MLX5_ACCEL_ESP_FLAGS_TUNNEL; 179 180 /* spi */ 181 attrs->spi = be32_to_cpu(x->id.spi); 182 183 /* source , destination ips */ 184 memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); 185 memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); 186 attrs->is_ipv6 = (x->props.family != AF_INET); 187 } 188 189 static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) 190 { 191 struct net_device *netdev = x->xso.real_dev; 192 struct mlx5e_priv *priv; 193 194 priv = netdev_priv(netdev); 195 196 if (x->props.aalgo != SADB_AALG_NONE) { 197 netdev_info(netdev, "Cannot offload authenticated xfrm states\n"); 198 return -EINVAL; 199 } 200 if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) { 201 netdev_info(netdev, "Only AES-GCM-ICV16 xfrm state may be offloaded\n"); 202 return -EINVAL; 203 } 204 if (x->props.calgo != SADB_X_CALG_NONE) { 205 netdev_info(netdev, "Cannot offload compressed xfrm states\n"); 206 return -EINVAL; 207 } 208 if (x->props.flags & XFRM_STATE_ESN && 209 !(mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_ESN)) { 210 netdev_info(netdev, "Cannot offload ESN xfrm states\n"); 211 return -EINVAL; 212 } 213 if (x->props.family != AF_INET && 214 x->props.family != AF_INET6) { 215 netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n"); 216 return -EINVAL; 217 } 218 if (x->props.mode != XFRM_MODE_TRANSPORT && 219 x->props.mode != XFRM_MODE_TUNNEL) { 220 dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n"); 221 return -EINVAL; 222 } 223 if (x->id.proto != IPPROTO_ESP) { 224 netdev_info(netdev, "Only ESP xfrm state may be offloaded\n"); 225 return -EINVAL; 226 } 227 if (x->encap) { 228 netdev_info(netdev, "Encapsulated xfrm state may not be offloaded\n"); 229 return -EINVAL; 230 } 231 if (!x->aead) { 232 netdev_info(netdev, "Cannot offload xfrm states without aead\n"); 233 return -EINVAL; 234 } 235 if (x->aead->alg_icv_len != 128) { 236 netdev_info(netdev, "Cannot offload xfrm states with AEAD ICV length other than 128bit\n"); 237 return -EINVAL; 238 } 239 if ((x->aead->alg_key_len != 128 + 32) && 240 (x->aead->alg_key_len != 256 + 32)) { 241 netdev_info(netdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n"); 242 return -EINVAL; 243 } 244 if (x->tfcpad) { 245 netdev_info(netdev, "Cannot offload xfrm states with tfc padding\n"); 246 return -EINVAL; 247 } 248 if (!x->geniv) { 249 netdev_info(netdev, "Cannot offload xfrm states without geniv\n"); 250 return -EINVAL; 251 } 252 if (strcmp(x->geniv, "seqiv")) { 253 netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n"); 254 return -EINVAL; 255 } 256 return 0; 257 } 258 259 static void _update_xfrm_state(struct work_struct *work) 260 { 261 struct mlx5e_ipsec_modify_state_work *modify_work = 262 container_of(work, struct mlx5e_ipsec_modify_state_work, work); 263 struct mlx5e_ipsec_sa_entry *sa_entry = container_of( 264 modify_work, struct mlx5e_ipsec_sa_entry, modify_work); 265 266 mlx5_accel_esp_modify_xfrm(sa_entry, &modify_work->attrs); 267 } 268 269 static int mlx5e_xfrm_add_state(struct xfrm_state *x) 270 { 271 struct mlx5e_ipsec_sa_entry *sa_entry = NULL; 272 struct net_device *netdev = x->xso.real_dev; 273 struct mlx5e_priv *priv; 274 int err; 275 276 priv = netdev_priv(netdev); 277 if (!priv->ipsec) 278 return -EOPNOTSUPP; 279 280 err = mlx5e_xfrm_validate_state(x); 281 if (err) 282 return err; 283 284 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); 285 if (!sa_entry) { 286 err = -ENOMEM; 287 goto out; 288 } 289 290 sa_entry->x = x; 291 sa_entry->ipsec = priv->ipsec; 292 293 /* check esn */ 294 mlx5e_ipsec_update_esn_state(sa_entry); 295 296 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs); 297 /* create hw context */ 298 err = mlx5_ipsec_create_sa_ctx(sa_entry); 299 if (err) 300 goto err_xfrm; 301 302 err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry); 303 if (err) 304 goto err_hw_ctx; 305 306 if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) { 307 err = mlx5e_ipsec_sadb_rx_add(sa_entry); 308 if (err) 309 goto err_add_rule; 310 } else { 311 sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? 312 mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; 313 } 314 315 INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state); 316 x->xso.offload_handle = (unsigned long)sa_entry; 317 goto out; 318 319 err_add_rule: 320 mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); 321 err_hw_ctx: 322 mlx5_ipsec_free_sa_ctx(sa_entry); 323 err_xfrm: 324 kfree(sa_entry); 325 out: 326 return err; 327 } 328 329 static void mlx5e_xfrm_del_state(struct xfrm_state *x) 330 { 331 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 332 333 if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) 334 mlx5e_ipsec_sadb_rx_del(sa_entry); 335 } 336 337 static void mlx5e_xfrm_free_state(struct xfrm_state *x) 338 { 339 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 340 struct mlx5e_priv *priv = netdev_priv(x->xso.dev); 341 342 cancel_work_sync(&sa_entry->modify_work.work); 343 mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); 344 mlx5_ipsec_free_sa_ctx(sa_entry); 345 kfree(sa_entry); 346 } 347 348 int mlx5e_ipsec_init(struct mlx5e_priv *priv) 349 { 350 struct mlx5e_ipsec *ipsec; 351 int ret; 352 353 if (!mlx5_ipsec_device_caps(priv->mdev)) { 354 netdev_dbg(priv->netdev, "Not an IPSec offload device\n"); 355 return 0; 356 } 357 358 ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL); 359 if (!ipsec) 360 return -ENOMEM; 361 362 hash_init(ipsec->sadb_rx); 363 spin_lock_init(&ipsec->sadb_rx_lock); 364 ipsec->mdev = priv->mdev; 365 ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, 366 priv->netdev->name); 367 if (!ipsec->wq) { 368 ret = -ENOMEM; 369 goto err_wq; 370 } 371 372 ret = mlx5e_accel_ipsec_fs_init(ipsec); 373 if (ret) 374 goto err_fs_init; 375 376 priv->ipsec = ipsec; 377 netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); 378 return 0; 379 380 err_fs_init: 381 destroy_workqueue(ipsec->wq); 382 err_wq: 383 kfree(ipsec); 384 return (ret != -EOPNOTSUPP) ? ret : 0; 385 } 386 387 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 388 { 389 struct mlx5e_ipsec *ipsec = priv->ipsec; 390 391 if (!ipsec) 392 return; 393 394 mlx5e_accel_ipsec_fs_cleanup(ipsec); 395 destroy_workqueue(ipsec->wq); 396 kfree(ipsec); 397 priv->ipsec = NULL; 398 } 399 400 static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 401 { 402 if (x->props.family == AF_INET) { 403 /* Offload with IPv4 options is not supported yet */ 404 if (ip_hdr(skb)->ihl > 5) 405 return false; 406 } else { 407 /* Offload with IPv6 extension headers is not support yet */ 408 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)) 409 return false; 410 } 411 412 return true; 413 } 414 415 static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) 416 { 417 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); 418 struct mlx5e_ipsec_modify_state_work *modify_work = 419 &sa_entry->modify_work; 420 bool need_update; 421 422 need_update = mlx5e_ipsec_update_esn_state(sa_entry); 423 if (!need_update) 424 return; 425 426 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs); 427 queue_work(sa_entry->ipsec->wq, &modify_work->work); 428 } 429 430 static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { 431 .xdo_dev_state_add = mlx5e_xfrm_add_state, 432 .xdo_dev_state_delete = mlx5e_xfrm_del_state, 433 .xdo_dev_state_free = mlx5e_xfrm_free_state, 434 .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, 435 .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, 436 }; 437 438 void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 439 { 440 struct mlx5_core_dev *mdev = priv->mdev; 441 struct net_device *netdev = priv->netdev; 442 443 if (!mlx5_ipsec_device_caps(mdev)) 444 return; 445 446 mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n"); 447 netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; 448 netdev->features |= NETIF_F_HW_ESP; 449 netdev->hw_enc_features |= NETIF_F_HW_ESP; 450 451 if (!MLX5_CAP_ETH(mdev, swp_csum)) { 452 mlx5_core_dbg(mdev, "mlx5e: SWP checksum not supported\n"); 453 return; 454 } 455 456 netdev->features |= NETIF_F_HW_ESP_TX_CSUM; 457 netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM; 458 459 if (!MLX5_CAP_ETH(mdev, swp_lso)) { 460 mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n"); 461 return; 462 } 463 464 netdev->gso_partial_features |= NETIF_F_GSO_ESP; 465 mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n"); 466 netdev->features |= NETIF_F_GSO_ESP; 467 netdev->hw_features |= NETIF_F_GSO_ESP; 468 netdev->hw_enc_features |= NETIF_F_GSO_ESP; 469 } 470