1 /* 2 * xfrm_device.c - IPsec device offloading code. 3 * 4 * Copyright (c) 2015 secunet Security Networks AG 5 * 6 * Author: 7 * Steffen Klassert <steffen.klassert@secunet.com> 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 12 * 2 of the License, or (at your option) any later version. 13 */ 14 15 #include <linux/errno.h> 16 #include <linux/module.h> 17 #include <linux/netdevice.h> 18 #include <linux/skbuff.h> 19 #include <linux/slab.h> 20 #include <linux/spinlock.h> 21 #include <net/dst.h> 22 #include <net/xfrm.h> 23 #include <linux/notifier.h> 24 25 #ifdef CONFIG_XFRM_OFFLOAD 26 struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again) 27 { 28 int err; 29 unsigned long flags; 30 struct xfrm_state *x; 31 struct sk_buff *skb2; 32 struct softnet_data *sd; 33 netdev_features_t esp_features = features; 34 struct xfrm_offload *xo = xfrm_offload(skb); 35 struct sec_path *sp; 36 37 if (!xo) 38 return skb; 39 40 if (!(features & NETIF_F_HW_ESP)) 41 esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK); 42 43 sp = skb_sec_path(skb); 44 x = sp->xvec[sp->len - 1]; 45 if (xo->flags & XFRM_GRO || x->xso.flags & XFRM_OFFLOAD_INBOUND) 46 return skb; 47 48 local_irq_save(flags); 49 sd = this_cpu_ptr(&softnet_data); 50 err = !skb_queue_empty(&sd->xfrm_backlog); 51 local_irq_restore(flags); 52 53 if (err) { 54 *again = true; 55 return skb; 56 } 57 58 if (skb_is_gso(skb)) { 59 struct net_device *dev = skb->dev; 60 61 if (unlikely(x->xso.dev != dev)) { 62 struct sk_buff *segs; 63 64 /* Packet got rerouted, fixup features and segment it. */ 65 esp_features = esp_features & ~(NETIF_F_HW_ESP 66 | NETIF_F_GSO_ESP); 67 68 segs = skb_gso_segment(skb, esp_features); 69 if (IS_ERR(segs)) { 70 kfree_skb(skb); 71 atomic_long_inc(&dev->tx_dropped); 72 return NULL; 73 } else { 74 consume_skb(skb); 75 skb = segs; 76 } 77 } 78 } 79 80 if (!skb->next) { 81 esp_features |= skb->dev->gso_partial_features; 82 x->outer_mode->xmit(x, skb); 83 84 xo->flags |= XFRM_DEV_RESUME; 85 86 err = x->type_offload->xmit(x, skb, esp_features); 87 if (err) { 88 if (err == -EINPROGRESS) 89 return NULL; 90 91 XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); 92 kfree_skb(skb); 93 return NULL; 94 } 95 96 skb_push(skb, skb->data - skb_mac_header(skb)); 97 98 return skb; 99 } 100 101 skb2 = skb; 102 103 do { 104 struct sk_buff *nskb = skb2->next; 105 106 esp_features |= skb->dev->gso_partial_features; 107 skb_mark_not_on_list(skb2); 108 109 xo = xfrm_offload(skb2); 110 xo->flags |= XFRM_DEV_RESUME; 111 112 x->outer_mode->xmit(x, skb2); 113 114 err = x->type_offload->xmit(x, skb2, esp_features); 115 if (!err) { 116 skb2->next = nskb; 117 } else if (err != -EINPROGRESS) { 118 XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); 119 skb2->next = nskb; 120 kfree_skb_list(skb2); 121 return NULL; 122 } else { 123 if (skb == skb2) 124 skb = nskb; 125 126 if (!skb) 127 return NULL; 128 129 goto skip_push; 130 } 131 132 skb_push(skb2, skb2->data - skb_mac_header(skb2)); 133 134 skip_push: 135 skb2 = nskb; 136 } while (skb2); 137 138 return skb; 139 } 140 EXPORT_SYMBOL_GPL(validate_xmit_xfrm); 141 142 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, 143 struct xfrm_user_offload *xuo) 144 { 145 int err; 146 struct dst_entry *dst; 147 struct net_device *dev; 148 struct xfrm_state_offload *xso = &x->xso; 149 xfrm_address_t *saddr; 150 xfrm_address_t *daddr; 151 152 if (!x->type_offload) 153 return -EINVAL; 154 155 /* We don't yet support UDP encapsulation and TFC padding. */ 156 if (x->encap || x->tfcpad) 157 return -EINVAL; 158 159 dev = dev_get_by_index(net, xuo->ifindex); 160 if (!dev) { 161 if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) { 162 saddr = &x->props.saddr; 163 daddr = &x->id.daddr; 164 } else { 165 saddr = &x->id.daddr; 166 daddr = &x->props.saddr; 167 } 168 169 dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr, 170 x->props.family, 171 xfrm_smark_get(0, x)); 172 if (IS_ERR(dst)) 173 return 0; 174 175 dev = dst->dev; 176 177 dev_hold(dev); 178 dst_release(dst); 179 } 180 181 if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) { 182 xso->dev = NULL; 183 dev_put(dev); 184 return 0; 185 } 186 187 if (x->props.flags & XFRM_STATE_ESN && 188 !dev->xfrmdev_ops->xdo_dev_state_advance_esn) { 189 xso->dev = NULL; 190 dev_put(dev); 191 return -EINVAL; 192 } 193 194 xso->dev = dev; 195 xso->num_exthdrs = 1; 196 xso->flags = xuo->flags; 197 198 err = dev->xfrmdev_ops->xdo_dev_state_add(x); 199 if (err) { 200 xso->num_exthdrs = 0; 201 xso->flags = 0; 202 xso->dev = NULL; 203 dev_put(dev); 204 205 if (err != -EOPNOTSUPP) 206 return err; 207 } 208 209 return 0; 210 } 211 EXPORT_SYMBOL_GPL(xfrm_dev_state_add); 212 213 bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 214 { 215 int mtu; 216 struct dst_entry *dst = skb_dst(skb); 217 struct xfrm_dst *xdst = (struct xfrm_dst *)dst; 218 struct net_device *dev = x->xso.dev; 219 220 if (!x->type_offload || x->encap) 221 return false; 222 223 if ((!dev || (dev == xfrm_dst_path(dst)->dev)) && 224 (!xdst->child->xfrm && x->type->get_mtu)) { 225 mtu = x->type->get_mtu(x, xdst->child_mtu_cached); 226 227 if (skb->len <= mtu) 228 goto ok; 229 230 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 231 goto ok; 232 } 233 234 return false; 235 236 ok: 237 if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok) 238 return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x); 239 240 return true; 241 } 242 EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok); 243 244 void xfrm_dev_resume(struct sk_buff *skb) 245 { 246 struct net_device *dev = skb->dev; 247 int ret = NETDEV_TX_BUSY; 248 struct netdev_queue *txq; 249 struct softnet_data *sd; 250 unsigned long flags; 251 252 rcu_read_lock(); 253 txq = netdev_core_pick_tx(dev, skb, NULL); 254 255 HARD_TX_LOCK(dev, txq, smp_processor_id()); 256 if (!netif_xmit_frozen_or_stopped(txq)) 257 skb = dev_hard_start_xmit(skb, dev, txq, &ret); 258 HARD_TX_UNLOCK(dev, txq); 259 260 if (!dev_xmit_complete(ret)) { 261 local_irq_save(flags); 262 sd = this_cpu_ptr(&softnet_data); 263 skb_queue_tail(&sd->xfrm_backlog, skb); 264 raise_softirq_irqoff(NET_TX_SOFTIRQ); 265 local_irq_restore(flags); 266 } 267 rcu_read_unlock(); 268 } 269 EXPORT_SYMBOL_GPL(xfrm_dev_resume); 270 271 void xfrm_dev_backlog(struct softnet_data *sd) 272 { 273 struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog; 274 struct sk_buff_head list; 275 struct sk_buff *skb; 276 277 if (skb_queue_empty(xfrm_backlog)) 278 return; 279 280 __skb_queue_head_init(&list); 281 282 spin_lock(&xfrm_backlog->lock); 283 skb_queue_splice_init(xfrm_backlog, &list); 284 spin_unlock(&xfrm_backlog->lock); 285 286 while (!skb_queue_empty(&list)) { 287 skb = __skb_dequeue(&list); 288 xfrm_dev_resume(skb); 289 } 290 291 } 292 #endif 293 294 static int xfrm_api_check(struct net_device *dev) 295 { 296 #ifdef CONFIG_XFRM_OFFLOAD 297 if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) && 298 !(dev->features & NETIF_F_HW_ESP)) 299 return NOTIFY_BAD; 300 301 if ((dev->features & NETIF_F_HW_ESP) && 302 (!(dev->xfrmdev_ops && 303 dev->xfrmdev_ops->xdo_dev_state_add && 304 dev->xfrmdev_ops->xdo_dev_state_delete))) 305 return NOTIFY_BAD; 306 #else 307 if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM)) 308 return NOTIFY_BAD; 309 #endif 310 311 return NOTIFY_DONE; 312 } 313 314 static int xfrm_dev_register(struct net_device *dev) 315 { 316 return xfrm_api_check(dev); 317 } 318 319 static int xfrm_dev_feat_change(struct net_device *dev) 320 { 321 return xfrm_api_check(dev); 322 } 323 324 static int xfrm_dev_down(struct net_device *dev) 325 { 326 if (dev->features & NETIF_F_HW_ESP) 327 xfrm_dev_state_flush(dev_net(dev), dev, true); 328 329 return NOTIFY_DONE; 330 } 331 332 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) 333 { 334 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 335 336 switch (event) { 337 case NETDEV_REGISTER: 338 return xfrm_dev_register(dev); 339 340 case NETDEV_FEAT_CHANGE: 341 return xfrm_dev_feat_change(dev); 342 343 case NETDEV_DOWN: 344 return xfrm_dev_down(dev); 345 } 346 return NOTIFY_DONE; 347 } 348 349 static struct notifier_block xfrm_dev_notifier = { 350 .notifier_call = xfrm_dev_event, 351 }; 352 353 void __init xfrm_dev_init(void) 354 { 355 register_netdevice_notifier(&xfrm_dev_notifier); 356 } 357