12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 221f42cc9SSteffen Klassert /* 321f42cc9SSteffen Klassert * xfrm_device.c - IPsec device offloading code. 421f42cc9SSteffen Klassert * 521f42cc9SSteffen Klassert * Copyright (c) 2015 secunet Security Networks AG 621f42cc9SSteffen Klassert * 721f42cc9SSteffen Klassert * Author: 821f42cc9SSteffen Klassert * Steffen Klassert <steffen.klassert@secunet.com> 921f42cc9SSteffen Klassert */ 1021f42cc9SSteffen Klassert 1121f42cc9SSteffen Klassert #include <linux/errno.h> 1221f42cc9SSteffen Klassert #include <linux/module.h> 1321f42cc9SSteffen Klassert #include <linux/netdevice.h> 1421f42cc9SSteffen Klassert #include <linux/skbuff.h> 1521f42cc9SSteffen Klassert #include <linux/slab.h> 1621f42cc9SSteffen Klassert #include <linux/spinlock.h> 1721f42cc9SSteffen Klassert #include <net/dst.h> 1821f42cc9SSteffen Klassert #include <net/xfrm.h> 1921f42cc9SSteffen Klassert #include <linux/notifier.h> 2021f42cc9SSteffen Klassert 21b81f884aSHangbin Liu #ifdef CONFIG_XFRM_OFFLOAD 22303c5fabSFlorian Westphal static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb, 23303c5fabSFlorian Westphal unsigned int hsize) 24303c5fabSFlorian Westphal { 25303c5fabSFlorian Westphal struct xfrm_offload *xo = xfrm_offload(skb); 26303c5fabSFlorian Westphal 27303c5fabSFlorian Westphal skb_reset_mac_len(skb); 2806a0afcfSXin Long if (xo->flags & XFRM_GSO_SEGMENT) 29303c5fabSFlorian Westphal skb->transport_header -= x->props.header_len; 3006a0afcfSXin Long 3106a0afcfSXin Long pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len); 32303c5fabSFlorian Westphal } 33303c5fabSFlorian Westphal 34303c5fabSFlorian Westphal static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb, 35303c5fabSFlorian Westphal unsigned int hsize) 36303c5fabSFlorian Westphal 37303c5fabSFlorian Westphal { 38303c5fabSFlorian Westphal struct xfrm_offload *xo = xfrm_offload(skb); 39303c5fabSFlorian Westphal 40303c5fabSFlorian Westphal if (xo->flags & XFRM_GSO_SEGMENT) 41303c5fabSFlorian Westphal skb->transport_header = skb->network_header + hsize; 42303c5fabSFlorian Westphal 43303c5fabSFlorian Westphal skb_reset_mac_len(skb); 44303c5fabSFlorian Westphal pskb_pull(skb, skb->mac_len + x->props.header_len); 45303c5fabSFlorian Westphal } 46303c5fabSFlorian Westphal 4730849175SXin Long static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb, 4830849175SXin Long unsigned int hsize) 4930849175SXin Long { 5030849175SXin Long struct xfrm_offload *xo = xfrm_offload(skb); 5130849175SXin Long int phlen = 0; 5230849175SXin Long 5330849175SXin Long if (xo->flags & XFRM_GSO_SEGMENT) 5430849175SXin Long skb->transport_header = skb->network_header + hsize; 5530849175SXin Long 5630849175SXin Long skb_reset_mac_len(skb); 5730849175SXin Long if (x->sel.family != AF_INET6) { 5830849175SXin Long phlen = IPV4_BEET_PHMAXLEN; 5930849175SXin Long if (x->outer_mode.family == AF_INET6) 6030849175SXin Long phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr); 6130849175SXin Long } 6230849175SXin Long 6330849175SXin Long pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen)); 6430849175SXin Long } 6530849175SXin Long 66303c5fabSFlorian Westphal /* Adjust pointers into the packet when IPsec is done at layer2 */ 67303c5fabSFlorian Westphal static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb) 68303c5fabSFlorian Westphal { 69c9500d7bSFlorian Westphal switch (x->outer_mode.encap) { 70303c5fabSFlorian Westphal case XFRM_MODE_TUNNEL: 71c9500d7bSFlorian Westphal if (x->outer_mode.family == AF_INET) 72303c5fabSFlorian Westphal return __xfrm_mode_tunnel_prep(x, skb, 73303c5fabSFlorian Westphal sizeof(struct iphdr)); 74c9500d7bSFlorian Westphal if (x->outer_mode.family == AF_INET6) 75303c5fabSFlorian Westphal return __xfrm_mode_tunnel_prep(x, skb, 76303c5fabSFlorian Westphal sizeof(struct ipv6hdr)); 77303c5fabSFlorian Westphal break; 78303c5fabSFlorian Westphal case XFRM_MODE_TRANSPORT: 79c9500d7bSFlorian Westphal if (x->outer_mode.family == AF_INET) 80303c5fabSFlorian Westphal return __xfrm_transport_prep(x, skb, 81303c5fabSFlorian Westphal sizeof(struct iphdr)); 82c9500d7bSFlorian Westphal if (x->outer_mode.family == AF_INET6) 83303c5fabSFlorian Westphal return __xfrm_transport_prep(x, skb, 84303c5fabSFlorian Westphal sizeof(struct ipv6hdr)); 85303c5fabSFlorian Westphal break; 8630849175SXin Long case XFRM_MODE_BEET: 8730849175SXin Long if (x->outer_mode.family == AF_INET) 8830849175SXin Long return __xfrm_mode_beet_prep(x, skb, 8930849175SXin Long sizeof(struct iphdr)); 9030849175SXin Long if (x->outer_mode.family == AF_INET6) 9130849175SXin Long return __xfrm_mode_beet_prep(x, skb, 9230849175SXin Long sizeof(struct ipv6hdr)); 9330849175SXin Long break; 94303c5fabSFlorian Westphal case XFRM_MODE_ROUTEOPTIMIZATION: 95303c5fabSFlorian Westphal case XFRM_MODE_IN_TRIGGER: 96303c5fabSFlorian Westphal break; 97303c5fabSFlorian Westphal } 98303c5fabSFlorian Westphal } 99303c5fabSFlorian Westphal 1004b549cccSChristian Langrock static inline bool xmit_xfrm_check_overflow(struct sk_buff *skb) 1014b549cccSChristian Langrock { 1024b549cccSChristian Langrock struct xfrm_offload *xo = xfrm_offload(skb); 1034b549cccSChristian Langrock __u32 seq = xo->seq.low; 1044b549cccSChristian Langrock 1054b549cccSChristian Langrock seq += skb_shinfo(skb)->gso_segs; 1064b549cccSChristian Langrock if (unlikely(seq < xo->seq.low)) 1074b549cccSChristian Langrock return true; 1084b549cccSChristian Langrock 1094b549cccSChristian Langrock return false; 1104b549cccSChristian Langrock } 1114b549cccSChristian Langrock 112f53c7239SSteffen Klassert struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again) 113f6e27114SSteffen Klassert { 114f6e27114SSteffen Klassert int err; 115f53c7239SSteffen Klassert unsigned long flags; 116f6e27114SSteffen Klassert struct xfrm_state *x; 117f53c7239SSteffen Klassert struct softnet_data *sd; 118d1d17a35SXin Long struct sk_buff *skb2, *nskb, *pskb = NULL; 1193dca3f38SSteffen Klassert netdev_features_t esp_features = features; 120f6e27114SSteffen Klassert struct xfrm_offload *xo = xfrm_offload(skb); 121272c2330SJarod Wilson struct net_device *dev = skb->dev; 1222294be0fSFlorian Westphal struct sec_path *sp; 123f6e27114SSteffen Klassert 12494579ac3SHuy Nguyen if (!xo || (xo->flags & XFRM_XMIT)) 1253dca3f38SSteffen Klassert return skb; 126f6e27114SSteffen Klassert 1273dca3f38SSteffen Klassert if (!(features & NETIF_F_HW_ESP)) 1283dca3f38SSteffen Klassert esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK); 1293dca3f38SSteffen Klassert 1302294be0fSFlorian Westphal sp = skb_sec_path(skb); 1312294be0fSFlorian Westphal x = sp->xvec[sp->len - 1]; 132482db2f1SLeon Romanovsky if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN) 1333dca3f38SSteffen Klassert return skb; 134f6e27114SSteffen Klassert 135bdfd2d1fSJarod Wilson /* This skb was already validated on the upper/virtual dev */ 136bdfd2d1fSJarod Wilson if ((x->xso.dev != dev) && (x->xso.real_dev == dev)) 137272c2330SJarod Wilson return skb; 138272c2330SJarod Wilson 139f53c7239SSteffen Klassert local_irq_save(flags); 140f53c7239SSteffen Klassert sd = this_cpu_ptr(&softnet_data); 141f53c7239SSteffen Klassert err = !skb_queue_empty(&sd->xfrm_backlog); 142f53c7239SSteffen Klassert local_irq_restore(flags); 143f53c7239SSteffen Klassert 144f53c7239SSteffen Klassert if (err) { 145f53c7239SSteffen Klassert *again = true; 146f53c7239SSteffen Klassert return skb; 147f53c7239SSteffen Klassert } 148f53c7239SSteffen Klassert 1494b549cccSChristian Langrock if (skb_is_gso(skb) && (unlikely(x->xso.dev != dev) || 1504b549cccSChristian Langrock unlikely(xmit_xfrm_check_overflow(skb)))) { 1513dca3f38SSteffen Klassert struct sk_buff *segs; 1523dca3f38SSteffen Klassert 1533dca3f38SSteffen Klassert /* Packet got rerouted, fixup features and segment it. */ 154272c2330SJarod Wilson esp_features = esp_features & ~(NETIF_F_HW_ESP | NETIF_F_GSO_ESP); 1553dca3f38SSteffen Klassert 1563dca3f38SSteffen Klassert segs = skb_gso_segment(skb, esp_features); 1573dca3f38SSteffen Klassert if (IS_ERR(segs)) { 1583dca3f38SSteffen Klassert kfree_skb(skb); 159625788b5SEric Dumazet dev_core_stats_tx_dropped_inc(dev); 1603dca3f38SSteffen Klassert return NULL; 1613dca3f38SSteffen Klassert } else { 1623dca3f38SSteffen Klassert consume_skb(skb); 1633dca3f38SSteffen Klassert skb = segs; 1643dca3f38SSteffen Klassert } 1653dca3f38SSteffen Klassert } 1663dca3f38SSteffen Klassert 1673dca3f38SSteffen Klassert if (!skb->next) { 16865fd2c2aSBoris Pismenny esp_features |= skb->dev->gso_partial_features; 169303c5fabSFlorian Westphal xfrm_outer_mode_prep(x, skb); 170f6e27114SSteffen Klassert 171f53c7239SSteffen Klassert xo->flags |= XFRM_DEV_RESUME; 172f53c7239SSteffen Klassert 1733dca3f38SSteffen Klassert err = x->type_offload->xmit(x, skb, esp_features); 174f6e27114SSteffen Klassert if (err) { 175f53c7239SSteffen Klassert if (err == -EINPROGRESS) 176f53c7239SSteffen Klassert return NULL; 177f53c7239SSteffen Klassert 178f6e27114SSteffen Klassert XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); 1793dca3f38SSteffen Klassert kfree_skb(skb); 1803dca3f38SSteffen Klassert return NULL; 181f6e27114SSteffen Klassert } 182f6e27114SSteffen Klassert 183f6e27114SSteffen Klassert skb_push(skb, skb->data - skb_mac_header(skb)); 1843dca3f38SSteffen Klassert 1853dca3f38SSteffen Klassert return skb; 186f6e27114SSteffen Klassert } 187f6e27114SSteffen Klassert 188c3b18e0dSJason A. Donenfeld skb_list_walk_safe(skb, skb2, nskb) { 18965fd2c2aSBoris Pismenny esp_features |= skb->dev->gso_partial_features; 190a8305bffSDavid S. Miller skb_mark_not_on_list(skb2); 1913dca3f38SSteffen Klassert 1923dca3f38SSteffen Klassert xo = xfrm_offload(skb2); 193f53c7239SSteffen Klassert xo->flags |= XFRM_DEV_RESUME; 1943dca3f38SSteffen Klassert 195303c5fabSFlorian Westphal xfrm_outer_mode_prep(x, skb2); 1963dca3f38SSteffen Klassert 1973dca3f38SSteffen Klassert err = x->type_offload->xmit(x, skb2, esp_features); 198f53c7239SSteffen Klassert if (!err) { 199f53c7239SSteffen Klassert skb2->next = nskb; 200f53c7239SSteffen Klassert } else if (err != -EINPROGRESS) { 2013dca3f38SSteffen Klassert XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); 2023dca3f38SSteffen Klassert skb2->next = nskb; 2033dca3f38SSteffen Klassert kfree_skb_list(skb2); 2043dca3f38SSteffen Klassert return NULL; 205f53c7239SSteffen Klassert } else { 206f53c7239SSteffen Klassert if (skb == skb2) 207f53c7239SSteffen Klassert skb = nskb; 208d1d17a35SXin Long else 209d1d17a35SXin Long pskb->next = nskb; 210f53c7239SSteffen Klassert 211c3b18e0dSJason A. Donenfeld continue; 212f53c7239SSteffen Klassert } 2133dca3f38SSteffen Klassert 2143dca3f38SSteffen Klassert skb_push(skb2, skb2->data - skb_mac_header(skb2)); 215d1d17a35SXin Long pskb = skb2; 216c3b18e0dSJason A. Donenfeld } 2173dca3f38SSteffen Klassert 2183dca3f38SSteffen Klassert return skb; 219f6e27114SSteffen Klassert } 220f6e27114SSteffen Klassert EXPORT_SYMBOL_GPL(validate_xmit_xfrm); 221f6e27114SSteffen Klassert 222d77e38e6SSteffen Klassert int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, 223adb5c33eSSabrina Dubroca struct xfrm_user_offload *xuo, 224adb5c33eSSabrina Dubroca struct netlink_ext_ack *extack) 225d77e38e6SSteffen Klassert { 226d77e38e6SSteffen Klassert int err; 227d77e38e6SSteffen Klassert struct dst_entry *dst; 228d77e38e6SSteffen Klassert struct net_device *dev; 22987e0a94eSLeon Romanovsky struct xfrm_dev_offload *xso = &x->xso; 230d77e38e6SSteffen Klassert xfrm_address_t *saddr; 231d77e38e6SSteffen Klassert xfrm_address_t *daddr; 232d77e38e6SSteffen Klassert 233adb5c33eSSabrina Dubroca if (!x->type_offload) { 234adb5c33eSSabrina Dubroca NL_SET_ERR_MSG(extack, "Type doesn't support offload"); 235ffdb5211SIlan Tayari return -EINVAL; 236adb5c33eSSabrina Dubroca } 237d77e38e6SSteffen Klassert 23850bd870aSYossef Efraim /* We don't yet support UDP encapsulation and TFC padding. */ 239adb5c33eSSabrina Dubroca if (x->encap || x->tfcpad) { 240adb5c33eSSabrina Dubroca NL_SET_ERR_MSG(extack, "Encapsulation and TFC padding can't be offloaded"); 24143024b9cSYossef Efraim return -EINVAL; 242adb5c33eSSabrina Dubroca } 243d77e38e6SSteffen Klassert 244adb5c33eSSabrina Dubroca if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) { 245adb5c33eSSabrina Dubroca NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request"); 2467c76ecd9SLeon Romanovsky return -EINVAL; 247adb5c33eSSabrina Dubroca } 2487c76ecd9SLeon Romanovsky 249d77e38e6SSteffen Klassert dev = dev_get_by_index(net, xuo->ifindex); 250d77e38e6SSteffen Klassert if (!dev) { 251d77e38e6SSteffen Klassert if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) { 252d77e38e6SSteffen Klassert saddr = &x->props.saddr; 253d77e38e6SSteffen Klassert daddr = &x->id.daddr; 254d77e38e6SSteffen Klassert } else { 255d77e38e6SSteffen Klassert saddr = &x->id.daddr; 256d77e38e6SSteffen Klassert daddr = &x->props.saddr; 257d77e38e6SSteffen Klassert } 258d77e38e6SSteffen Klassert 259077fbac4SLorenzo Colitti dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr, 2609b42c1f1SSteffen Klassert x->props.family, 2619b42c1f1SSteffen Klassert xfrm_smark_get(0, x)); 262d77e38e6SSteffen Klassert if (IS_ERR(dst)) 263d77e38e6SSteffen Klassert return 0; 264d77e38e6SSteffen Klassert 265d77e38e6SSteffen Klassert dev = dst->dev; 266d77e38e6SSteffen Klassert 267d77e38e6SSteffen Klassert dev_hold(dev); 268d77e38e6SSteffen Klassert dst_release(dst); 269d77e38e6SSteffen Klassert } 270d77e38e6SSteffen Klassert 271d77e38e6SSteffen Klassert if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) { 27267a63387SSteffen Klassert xso->dev = NULL; 273d77e38e6SSteffen Klassert dev_put(dev); 274d77e38e6SSteffen Klassert return 0; 275d77e38e6SSteffen Klassert } 276d77e38e6SSteffen Klassert 27750bd870aSYossef Efraim if (x->props.flags & XFRM_STATE_ESN && 27850bd870aSYossef Efraim !dev->xfrmdev_ops->xdo_dev_state_advance_esn) { 279adb5c33eSSabrina Dubroca NL_SET_ERR_MSG(extack, "Device doesn't support offload with ESN"); 28050bd870aSYossef Efraim xso->dev = NULL; 28150bd870aSYossef Efraim dev_put(dev); 28250bd870aSYossef Efraim return -EINVAL; 28350bd870aSYossef Efraim } 28450bd870aSYossef Efraim 285d77e38e6SSteffen Klassert xso->dev = dev; 286e1b539bdSEric Dumazet netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC); 287bdfd2d1fSJarod Wilson xso->real_dev = dev; 288d77e38e6SSteffen Klassert 289482db2f1SLeon Romanovsky if (xuo->flags & XFRM_OFFLOAD_INBOUND) 290482db2f1SLeon Romanovsky xso->dir = XFRM_DEV_OFFLOAD_IN; 291482db2f1SLeon Romanovsky else 292482db2f1SLeon Romanovsky xso->dir = XFRM_DEV_OFFLOAD_OUT; 293482db2f1SLeon Romanovsky 294*d14f28b8SLeon Romanovsky xso->type = XFRM_DEV_OFFLOAD_CRYPTO; 295*d14f28b8SLeon Romanovsky 296d77e38e6SSteffen Klassert err = dev->xfrmdev_ops->xdo_dev_state_add(x); 297d77e38e6SSteffen Klassert if (err) { 298aa5dd6faSAviad Yehezkel xso->dev = NULL; 299482db2f1SLeon Romanovsky xso->dir = 0; 300dd72fadfSAyush Sawal xso->real_dev = NULL; 301d62607c3SJakub Kicinski netdev_put(dev, &xso->dev_tracker); 302*d14f28b8SLeon Romanovsky xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED; 3034a132095SShannon Nelson 304adb5c33eSSabrina Dubroca if (err != -EOPNOTSUPP) { 305adb5c33eSSabrina Dubroca NL_SET_ERR_MSG(extack, "Device failed to offload this state"); 306d77e38e6SSteffen Klassert return err; 307d77e38e6SSteffen Klassert } 308adb5c33eSSabrina Dubroca } 309d77e38e6SSteffen Klassert 310d77e38e6SSteffen Klassert return 0; 311d77e38e6SSteffen Klassert } 312d77e38e6SSteffen Klassert EXPORT_SYMBOL_GPL(xfrm_dev_state_add); 313d77e38e6SSteffen Klassert 314d77e38e6SSteffen Klassert bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 315d77e38e6SSteffen Klassert { 316d77e38e6SSteffen Klassert int mtu; 317d77e38e6SSteffen Klassert struct dst_entry *dst = skb_dst(skb); 318d77e38e6SSteffen Klassert struct xfrm_dst *xdst = (struct xfrm_dst *)dst; 319d77e38e6SSteffen Klassert struct net_device *dev = x->xso.dev; 320d77e38e6SSteffen Klassert 321d77e38e6SSteffen Klassert if (!x->type_offload || x->encap) 322d77e38e6SSteffen Klassert return false; 323d77e38e6SSteffen Klassert 324fcb662deSShannon Nelson if ((!dev || (dev == xfrm_dst_path(dst)->dev)) && 325c7b37c76SFlorian Westphal (!xdst->child->xfrm)) { 326c7b37c76SFlorian Westphal mtu = xfrm_state_mtu(x, xdst->child_mtu_cached); 327d77e38e6SSteffen Klassert if (skb->len <= mtu) 328d77e38e6SSteffen Klassert goto ok; 329d77e38e6SSteffen Klassert 330779b7931SDaniel Axtens if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 331d77e38e6SSteffen Klassert goto ok; 332d77e38e6SSteffen Klassert } 333d77e38e6SSteffen Klassert 334d77e38e6SSteffen Klassert return false; 335d77e38e6SSteffen Klassert 336d77e38e6SSteffen Klassert ok: 337d77e38e6SSteffen Klassert if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok) 338d77e38e6SSteffen Klassert return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x); 339d77e38e6SSteffen Klassert 340d77e38e6SSteffen Klassert return true; 341d77e38e6SSteffen Klassert } 342d77e38e6SSteffen Klassert EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok); 343f53c7239SSteffen Klassert 344f53c7239SSteffen Klassert void xfrm_dev_resume(struct sk_buff *skb) 345f53c7239SSteffen Klassert { 346f53c7239SSteffen Klassert struct net_device *dev = skb->dev; 347f53c7239SSteffen Klassert int ret = NETDEV_TX_BUSY; 348f53c7239SSteffen Klassert struct netdev_queue *txq; 349f53c7239SSteffen Klassert struct softnet_data *sd; 350f53c7239SSteffen Klassert unsigned long flags; 351f53c7239SSteffen Klassert 352f53c7239SSteffen Klassert rcu_read_lock(); 3534bd97d51SPaolo Abeni txq = netdev_core_pick_tx(dev, skb, NULL); 354f53c7239SSteffen Klassert 355f53c7239SSteffen Klassert HARD_TX_LOCK(dev, txq, smp_processor_id()); 356f53c7239SSteffen Klassert if (!netif_xmit_frozen_or_stopped(txq)) 357f53c7239SSteffen Klassert skb = dev_hard_start_xmit(skb, dev, txq, &ret); 358f53c7239SSteffen Klassert HARD_TX_UNLOCK(dev, txq); 359f53c7239SSteffen Klassert 360f53c7239SSteffen Klassert if (!dev_xmit_complete(ret)) { 361f53c7239SSteffen Klassert local_irq_save(flags); 362f53c7239SSteffen Klassert sd = this_cpu_ptr(&softnet_data); 363f53c7239SSteffen Klassert skb_queue_tail(&sd->xfrm_backlog, skb); 364f53c7239SSteffen Klassert raise_softirq_irqoff(NET_TX_SOFTIRQ); 365f53c7239SSteffen Klassert local_irq_restore(flags); 366f53c7239SSteffen Klassert } 367f53c7239SSteffen Klassert rcu_read_unlock(); 368f53c7239SSteffen Klassert } 369f53c7239SSteffen Klassert EXPORT_SYMBOL_GPL(xfrm_dev_resume); 370f53c7239SSteffen Klassert 371f53c7239SSteffen Klassert void xfrm_dev_backlog(struct softnet_data *sd) 372f53c7239SSteffen Klassert { 373f53c7239SSteffen Klassert struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog; 374f53c7239SSteffen Klassert struct sk_buff_head list; 375f53c7239SSteffen Klassert struct sk_buff *skb; 376f53c7239SSteffen Klassert 377f53c7239SSteffen Klassert if (skb_queue_empty(xfrm_backlog)) 378f53c7239SSteffen Klassert return; 379f53c7239SSteffen Klassert 380f53c7239SSteffen Klassert __skb_queue_head_init(&list); 381f53c7239SSteffen Klassert 382f53c7239SSteffen Klassert spin_lock(&xfrm_backlog->lock); 383f53c7239SSteffen Klassert skb_queue_splice_init(xfrm_backlog, &list); 384f53c7239SSteffen Klassert spin_unlock(&xfrm_backlog->lock); 385f53c7239SSteffen Klassert 386f53c7239SSteffen Klassert while (!skb_queue_empty(&list)) { 387f53c7239SSteffen Klassert skb = __skb_dequeue(&list); 388f53c7239SSteffen Klassert xfrm_dev_resume(skb); 389f53c7239SSteffen Klassert } 390f53c7239SSteffen Klassert 391f53c7239SSteffen Klassert } 392b81f884aSHangbin Liu #endif 393d77e38e6SSteffen Klassert 39492a23206SShannon Nelson static int xfrm_api_check(struct net_device *dev) 395d77e38e6SSteffen Klassert { 39692a23206SShannon Nelson #ifdef CONFIG_XFRM_OFFLOAD 397d77e38e6SSteffen Klassert if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) && 398d77e38e6SSteffen Klassert !(dev->features & NETIF_F_HW_ESP)) 399d77e38e6SSteffen Klassert return NOTIFY_BAD; 400d77e38e6SSteffen Klassert 40192a23206SShannon Nelson if ((dev->features & NETIF_F_HW_ESP) && 40292a23206SShannon Nelson (!(dev->xfrmdev_ops && 40392a23206SShannon Nelson dev->xfrmdev_ops->xdo_dev_state_add && 40492a23206SShannon Nelson dev->xfrmdev_ops->xdo_dev_state_delete))) 40592a23206SShannon Nelson return NOTIFY_BAD; 40692a23206SShannon Nelson #else 40792a23206SShannon Nelson if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM)) 40892a23206SShannon Nelson return NOTIFY_BAD; 40992a23206SShannon Nelson #endif 41092a23206SShannon Nelson 411d77e38e6SSteffen Klassert return NOTIFY_DONE; 412d77e38e6SSteffen Klassert } 413d77e38e6SSteffen Klassert 414d77e38e6SSteffen Klassert static int xfrm_dev_down(struct net_device *dev) 415d77e38e6SSteffen Klassert { 4162c1497bbSIlan Tayari if (dev->features & NETIF_F_HW_ESP) 417d77e38e6SSteffen Klassert xfrm_dev_state_flush(dev_net(dev), dev, true); 418d77e38e6SSteffen Klassert 419d77e38e6SSteffen Klassert return NOTIFY_DONE; 420d77e38e6SSteffen Klassert } 421d77e38e6SSteffen Klassert 42221f42cc9SSteffen Klassert static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) 42321f42cc9SSteffen Klassert { 42421f42cc9SSteffen Klassert struct net_device *dev = netdev_notifier_info_to_dev(ptr); 42521f42cc9SSteffen Klassert 42621f42cc9SSteffen Klassert switch (event) { 427d77e38e6SSteffen Klassert case NETDEV_REGISTER: 4282ecda181SLeon Romanovsky return xfrm_api_check(dev); 429d77e38e6SSteffen Klassert 430d77e38e6SSteffen Klassert case NETDEV_FEAT_CHANGE: 4312ecda181SLeon Romanovsky return xfrm_api_check(dev); 432d77e38e6SSteffen Klassert 43321f42cc9SSteffen Klassert case NETDEV_DOWN: 43403891f82SRaed Salem case NETDEV_UNREGISTER: 435d77e38e6SSteffen Klassert return xfrm_dev_down(dev); 43621f42cc9SSteffen Klassert } 43721f42cc9SSteffen Klassert return NOTIFY_DONE; 43821f42cc9SSteffen Klassert } 43921f42cc9SSteffen Klassert 44021f42cc9SSteffen Klassert static struct notifier_block xfrm_dev_notifier = { 44121f42cc9SSteffen Klassert .notifier_call = xfrm_dev_event, 44221f42cc9SSteffen Klassert }; 44321f42cc9SSteffen Klassert 444e9a441b6SKirill Tkhai void __init xfrm_dev_init(void) 44521f42cc9SSteffen Klassert { 44621f42cc9SSteffen Klassert register_netdevice_notifier(&xfrm_dev_notifier); 44721f42cc9SSteffen Klassert } 448