1 /* 2 * xfrm_device.c - IPsec device offloading code. 3 * 4 * Copyright (c) 2015 secunet Security Networks AG 5 * 6 * Author: 7 * Steffen Klassert <steffen.klassert@secunet.com> 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 12 * 2 of the License, or (at your option) any later version. 13 */ 14 15 #include <linux/errno.h> 16 #include <linux/module.h> 17 #include <linux/netdevice.h> 18 #include <linux/skbuff.h> 19 #include <linux/slab.h> 20 #include <linux/spinlock.h> 21 #include <net/dst.h> 22 #include <net/xfrm.h> 23 #include <linux/notifier.h> 24 25 #ifdef CONFIG_XFRM_OFFLOAD 26 static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb, 27 unsigned int hsize) 28 { 29 struct xfrm_offload *xo = xfrm_offload(skb); 30 31 skb_reset_mac_len(skb); 32 pskb_pull(skb, skb->mac_len + hsize + x->props.header_len); 33 34 if (xo->flags & XFRM_GSO_SEGMENT) { 35 skb_reset_transport_header(skb); 36 skb->transport_header -= x->props.header_len; 37 } 38 } 39 40 static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb, 41 unsigned int hsize) 42 43 { 44 struct xfrm_offload *xo = xfrm_offload(skb); 45 46 if (xo->flags & XFRM_GSO_SEGMENT) 47 skb->transport_header = skb->network_header + hsize; 48 49 skb_reset_mac_len(skb); 50 pskb_pull(skb, skb->mac_len + x->props.header_len); 51 } 52 53 /* Adjust pointers into the packet when IPsec is done at layer2 */ 54 static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb) 55 { 56 switch (x->outer_mode.encap) { 57 case XFRM_MODE_TUNNEL: 58 if (x->outer_mode.family == AF_INET) 59 return __xfrm_mode_tunnel_prep(x, skb, 60 sizeof(struct iphdr)); 61 if (x->outer_mode.family == AF_INET6) 62 return __xfrm_mode_tunnel_prep(x, skb, 63 sizeof(struct ipv6hdr)); 64 break; 65 case XFRM_MODE_TRANSPORT: 66 if (x->outer_mode.family == AF_INET) 67 return __xfrm_transport_prep(x, skb, 68 sizeof(struct iphdr)); 69 if (x->outer_mode.family == AF_INET6) 70 return __xfrm_transport_prep(x, skb, 71 sizeof(struct ipv6hdr)); 72 break; 73 case XFRM_MODE_ROUTEOPTIMIZATION: 74 case XFRM_MODE_IN_TRIGGER: 75 case XFRM_MODE_BEET: 76 break; 77 } 78 } 79 80 struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again) 81 { 82 int err; 83 unsigned long flags; 84 struct xfrm_state *x; 85 struct sk_buff *skb2; 86 struct softnet_data *sd; 87 netdev_features_t esp_features = features; 88 struct xfrm_offload *xo = xfrm_offload(skb); 89 struct sec_path *sp; 90 91 if (!xo) 92 return skb; 93 94 if (!(features & NETIF_F_HW_ESP)) 95 esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK); 96 97 sp = skb_sec_path(skb); 98 x = sp->xvec[sp->len - 1]; 99 if (xo->flags & XFRM_GRO || x->xso.flags & XFRM_OFFLOAD_INBOUND) 100 return skb; 101 102 local_irq_save(flags); 103 sd = this_cpu_ptr(&softnet_data); 104 err = !skb_queue_empty(&sd->xfrm_backlog); 105 local_irq_restore(flags); 106 107 if (err) { 108 *again = true; 109 return skb; 110 } 111 112 if (skb_is_gso(skb)) { 113 struct net_device *dev = skb->dev; 114 115 if (unlikely(x->xso.dev != dev)) { 116 struct sk_buff *segs; 117 118 /* Packet got rerouted, fixup features and segment it. */ 119 esp_features = esp_features & ~(NETIF_F_HW_ESP 120 | NETIF_F_GSO_ESP); 121 122 segs = skb_gso_segment(skb, esp_features); 123 if (IS_ERR(segs)) { 124 kfree_skb(skb); 125 atomic_long_inc(&dev->tx_dropped); 126 return NULL; 127 } else { 128 consume_skb(skb); 129 skb = segs; 130 } 131 } 132 } 133 134 if (!skb->next) { 135 esp_features |= skb->dev->gso_partial_features; 136 xfrm_outer_mode_prep(x, skb); 137 138 xo->flags |= XFRM_DEV_RESUME; 139 140 err = x->type_offload->xmit(x, skb, esp_features); 141 if (err) { 142 if (err == -EINPROGRESS) 143 return NULL; 144 145 XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); 146 kfree_skb(skb); 147 return NULL; 148 } 149 150 skb_push(skb, skb->data - skb_mac_header(skb)); 151 152 return skb; 153 } 154 155 skb2 = skb; 156 157 do { 158 struct sk_buff *nskb = skb2->next; 159 160 esp_features |= skb->dev->gso_partial_features; 161 skb_mark_not_on_list(skb2); 162 163 xo = xfrm_offload(skb2); 164 xo->flags |= XFRM_DEV_RESUME; 165 166 xfrm_outer_mode_prep(x, skb2); 167 168 err = x->type_offload->xmit(x, skb2, esp_features); 169 if (!err) { 170 skb2->next = nskb; 171 } else if (err != -EINPROGRESS) { 172 XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); 173 skb2->next = nskb; 174 kfree_skb_list(skb2); 175 return NULL; 176 } else { 177 if (skb == skb2) 178 skb = nskb; 179 180 if (!skb) 181 return NULL; 182 183 goto skip_push; 184 } 185 186 skb_push(skb2, skb2->data - skb_mac_header(skb2)); 187 188 skip_push: 189 skb2 = nskb; 190 } while (skb2); 191 192 return skb; 193 } 194 EXPORT_SYMBOL_GPL(validate_xmit_xfrm); 195 196 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, 197 struct xfrm_user_offload *xuo) 198 { 199 int err; 200 struct dst_entry *dst; 201 struct net_device *dev; 202 struct xfrm_state_offload *xso = &x->xso; 203 xfrm_address_t *saddr; 204 xfrm_address_t *daddr; 205 206 if (!x->type_offload) 207 return -EINVAL; 208 209 /* We don't yet support UDP encapsulation and TFC padding. */ 210 if (x->encap || x->tfcpad) 211 return -EINVAL; 212 213 dev = dev_get_by_index(net, xuo->ifindex); 214 if (!dev) { 215 if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) { 216 saddr = &x->props.saddr; 217 daddr = &x->id.daddr; 218 } else { 219 saddr = &x->id.daddr; 220 daddr = &x->props.saddr; 221 } 222 223 dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr, 224 x->props.family, 225 xfrm_smark_get(0, x)); 226 if (IS_ERR(dst)) 227 return 0; 228 229 dev = dst->dev; 230 231 dev_hold(dev); 232 dst_release(dst); 233 } 234 235 if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) { 236 xso->dev = NULL; 237 dev_put(dev); 238 return 0; 239 } 240 241 if (x->props.flags & XFRM_STATE_ESN && 242 !dev->xfrmdev_ops->xdo_dev_state_advance_esn) { 243 xso->dev = NULL; 244 dev_put(dev); 245 return -EINVAL; 246 } 247 248 xso->dev = dev; 249 xso->num_exthdrs = 1; 250 xso->flags = xuo->flags; 251 252 err = dev->xfrmdev_ops->xdo_dev_state_add(x); 253 if (err) { 254 xso->num_exthdrs = 0; 255 xso->flags = 0; 256 xso->dev = NULL; 257 dev_put(dev); 258 259 if (err != -EOPNOTSUPP) 260 return err; 261 } 262 263 return 0; 264 } 265 EXPORT_SYMBOL_GPL(xfrm_dev_state_add); 266 267 bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 268 { 269 int mtu; 270 struct dst_entry *dst = skb_dst(skb); 271 struct xfrm_dst *xdst = (struct xfrm_dst *)dst; 272 struct net_device *dev = x->xso.dev; 273 274 if (!x->type_offload || x->encap) 275 return false; 276 277 if ((!dev || (dev == xfrm_dst_path(dst)->dev)) && 278 (!xdst->child->xfrm)) { 279 mtu = xfrm_state_mtu(x, xdst->child_mtu_cached); 280 if (skb->len <= mtu) 281 goto ok; 282 283 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) 284 goto ok; 285 } 286 287 return false; 288 289 ok: 290 if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok) 291 return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x); 292 293 return true; 294 } 295 EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok); 296 297 void xfrm_dev_resume(struct sk_buff *skb) 298 { 299 struct net_device *dev = skb->dev; 300 int ret = NETDEV_TX_BUSY; 301 struct netdev_queue *txq; 302 struct softnet_data *sd; 303 unsigned long flags; 304 305 rcu_read_lock(); 306 txq = netdev_core_pick_tx(dev, skb, NULL); 307 308 HARD_TX_LOCK(dev, txq, smp_processor_id()); 309 if (!netif_xmit_frozen_or_stopped(txq)) 310 skb = dev_hard_start_xmit(skb, dev, txq, &ret); 311 HARD_TX_UNLOCK(dev, txq); 312 313 if (!dev_xmit_complete(ret)) { 314 local_irq_save(flags); 315 sd = this_cpu_ptr(&softnet_data); 316 skb_queue_tail(&sd->xfrm_backlog, skb); 317 raise_softirq_irqoff(NET_TX_SOFTIRQ); 318 local_irq_restore(flags); 319 } 320 rcu_read_unlock(); 321 } 322 EXPORT_SYMBOL_GPL(xfrm_dev_resume); 323 324 void xfrm_dev_backlog(struct softnet_data *sd) 325 { 326 struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog; 327 struct sk_buff_head list; 328 struct sk_buff *skb; 329 330 if (skb_queue_empty(xfrm_backlog)) 331 return; 332 333 __skb_queue_head_init(&list); 334 335 spin_lock(&xfrm_backlog->lock); 336 skb_queue_splice_init(xfrm_backlog, &list); 337 spin_unlock(&xfrm_backlog->lock); 338 339 while (!skb_queue_empty(&list)) { 340 skb = __skb_dequeue(&list); 341 xfrm_dev_resume(skb); 342 } 343 344 } 345 #endif 346 347 static int xfrm_api_check(struct net_device *dev) 348 { 349 #ifdef CONFIG_XFRM_OFFLOAD 350 if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) && 351 !(dev->features & NETIF_F_HW_ESP)) 352 return NOTIFY_BAD; 353 354 if ((dev->features & NETIF_F_HW_ESP) && 355 (!(dev->xfrmdev_ops && 356 dev->xfrmdev_ops->xdo_dev_state_add && 357 dev->xfrmdev_ops->xdo_dev_state_delete))) 358 return NOTIFY_BAD; 359 #else 360 if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM)) 361 return NOTIFY_BAD; 362 #endif 363 364 return NOTIFY_DONE; 365 } 366 367 static int xfrm_dev_register(struct net_device *dev) 368 { 369 return xfrm_api_check(dev); 370 } 371 372 static int xfrm_dev_feat_change(struct net_device *dev) 373 { 374 return xfrm_api_check(dev); 375 } 376 377 static int xfrm_dev_down(struct net_device *dev) 378 { 379 if (dev->features & NETIF_F_HW_ESP) 380 xfrm_dev_state_flush(dev_net(dev), dev, true); 381 382 return NOTIFY_DONE; 383 } 384 385 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) 386 { 387 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 388 389 switch (event) { 390 case NETDEV_REGISTER: 391 return xfrm_dev_register(dev); 392 393 case NETDEV_FEAT_CHANGE: 394 return xfrm_dev_feat_change(dev); 395 396 case NETDEV_DOWN: 397 return xfrm_dev_down(dev); 398 } 399 return NOTIFY_DONE; 400 } 401 402 static struct notifier_block xfrm_dev_notifier = { 403 .notifier_call = xfrm_dev_event, 404 }; 405 406 void __init xfrm_dev_init(void) 407 { 408 register_netdevice_notifier(&xfrm_dev_notifier); 409 } 410