dev.c (386b55488849207a67a462a3a4ccc3dc8c7f21b9) dev.c (3d3ea5af5c0b382bc9d9aed378fd814fb5d4a011)
1/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *

--- 91 unchanged lines hidden (view full) ---

100#include <net/net_namespace.h>
101#include <net/sock.h>
102#include <net/busy_poll.h>
103#include <linux/rtnetlink.h>
104#include <linux/stat.h>
105#include <net/dst.h>
106#include <net/dst_metadata.h>
107#include <net/pkt_sched.h>
1/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *

--- 91 unchanged lines hidden (view full) ---

100#include <net/net_namespace.h>
101#include <net/sock.h>
102#include <net/busy_poll.h>
103#include <linux/rtnetlink.h>
104#include <linux/stat.h>
105#include <net/dst.h>
106#include <net/dst_metadata.h>
107#include <net/pkt_sched.h>
108#include <net/pkt_cls.h>
108#include <net/checksum.h>
109#include <net/xfrm.h>
110#include <linux/highmem.h>
111#include <linux/init.h>
112#include <linux/module.h>
113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>

--- 21 unchanged lines hidden (view full) ---

137#include <linux/static_key.h>
138#include <linux/hashtable.h>
139#include <linux/vmalloc.h>
140#include <linux/if_macvlan.h>
141#include <linux/errqueue.h>
142#include <linux/hrtimer.h>
143#include <linux/netfilter_ingress.h>
144#include <linux/crash_dump.h>
109#include <net/checksum.h>
110#include <net/xfrm.h>
111#include <linux/highmem.h>
112#include <linux/init.h>
113#include <linux/module.h>
114#include <linux/netpoll.h>
115#include <linux/rcupdate.h>
116#include <linux/delay.h>

--- 21 unchanged lines hidden (view full) ---

138#include <linux/static_key.h>
139#include <linux/hashtable.h>
140#include <linux/vmalloc.h>
141#include <linux/if_macvlan.h>
142#include <linux/errqueue.h>
143#include <linux/hrtimer.h>
144#include <linux/netfilter_ingress.h>
145#include <linux/crash_dump.h>
146#include <linux/sctp.h>
145
146#include "net-sysfs.h"
147
148/* Instead of increasing this, you should create a hash table. */
149#define MAX_GRO_SKBS 8
150
151/* This should be increased if a protocol with a bigger head is added. */
152#define GRO_MAX_HEAD (MAX_HEADER + 128)
153
154static DEFINE_SPINLOCK(ptype_lock);
155static DEFINE_SPINLOCK(offload_lock);
156struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
157struct list_head ptype_all __read_mostly; /* Taps */
158static struct list_head offload_base __read_mostly;
159
160static int netif_rx_internal(struct sk_buff *skb);
161static int call_netdevice_notifiers_info(unsigned long val,
162 struct net_device *dev,
163 struct netdev_notifier_info *info);
147
148#include "net-sysfs.h"
149
150/* Instead of increasing this, you should create a hash table. */
151#define MAX_GRO_SKBS 8
152
153/* This should be increased if a protocol with a bigger head is added. */
154#define GRO_MAX_HEAD (MAX_HEADER + 128)
155
156static DEFINE_SPINLOCK(ptype_lock);
157static DEFINE_SPINLOCK(offload_lock);
158struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
159struct list_head ptype_all __read_mostly; /* Taps */
160static struct list_head offload_base __read_mostly;
161
162static int netif_rx_internal(struct sk_buff *skb);
163static int call_netdevice_notifiers_info(unsigned long val,
164 struct net_device *dev,
165 struct netdev_notifier_info *info);
166static struct napi_struct *napi_by_id(unsigned int napi_id);
164
165/*
166 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
167 * semaphore.
168 *
169 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
170 *
171 * Writers must hold the rtnl semaphore while they loop through the

--- 688 unchanged lines hidden (view full) ---

860 if (dev)
861 dev_hold(dev);
862 rcu_read_unlock();
863 return dev;
864}
865EXPORT_SYMBOL(dev_get_by_index);
866
867/**
167
168/*
169 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
170 * semaphore.
171 *
172 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
173 *
174 * Writers must hold the rtnl semaphore while they loop through the

--- 688 unchanged lines hidden (view full) ---

863 if (dev)
864 dev_hold(dev);
865 rcu_read_unlock();
866 return dev;
867}
868EXPORT_SYMBOL(dev_get_by_index);
869
870/**
871 * dev_get_by_napi_id - find a device by napi_id
872 * @napi_id: ID of the NAPI struct
873 *
874 * Search for an interface by NAPI ID. Returns %NULL if the device
875 * is not found or a pointer to the device. The device has not had
876 * its reference counter increased so the caller must be careful
877 * about locking. The caller must hold RCU lock.
878 */
879
880struct net_device *dev_get_by_napi_id(unsigned int napi_id)
881{
882 struct napi_struct *napi;
883
884 WARN_ON_ONCE(!rcu_read_lock_held());
885
886 if (napi_id < MIN_NAPI_ID)
887 return NULL;
888
889 napi = napi_by_id(napi_id);
890
891 return napi ? napi->dev : NULL;
892}
893EXPORT_SYMBOL(dev_get_by_napi_id);
894
895/**
868 * netdev_get_name - get a netdevice name, knowing its ifindex.
869 * @net: network namespace
870 * @name: a pointer to the buffer where the name will be stored.
871 * @ifindex: the ifindex of the interface to get the name from.
872 *
873 * The use of raw_seqcount_begin() and cond_resched() before
874 * retrying is required as we want to give the writers a chance
875 * to complete when CONFIG_PREEMPT is not set.

--- 1730 unchanged lines hidden (view full) ---

2606 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2607out_set_summed:
2608 skb->ip_summed = CHECKSUM_NONE;
2609out:
2610 return ret;
2611}
2612EXPORT_SYMBOL(skb_checksum_help);
2613
896 * netdev_get_name - get a netdevice name, knowing its ifindex.
897 * @net: network namespace
898 * @name: a pointer to the buffer where the name will be stored.
899 * @ifindex: the ifindex of the interface to get the name from.
900 *
901 * The use of raw_seqcount_begin() and cond_resched() before
902 * retrying is required as we want to give the writers a chance
903 * to complete when CONFIG_PREEMPT is not set.

--- 1730 unchanged lines hidden (view full) ---

2634 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2635out_set_summed:
2636 skb->ip_summed = CHECKSUM_NONE;
2637out:
2638 return ret;
2639}
2640EXPORT_SYMBOL(skb_checksum_help);
2641
2642int skb_crc32c_csum_help(struct sk_buff *skb)
2643{
2644 __le32 crc32c_csum;
2645 int ret = 0, offset, start;
2646
2647 if (skb->ip_summed != CHECKSUM_PARTIAL)
2648 goto out;
2649
2650 if (unlikely(skb_is_gso(skb)))
2651 goto out;
2652
2653 /* Before computing a checksum, we should make sure no frag could
2654 * be modified by an external entity : checksum could be wrong.
2655 */
2656 if (unlikely(skb_has_shared_frag(skb))) {
2657 ret = __skb_linearize(skb);
2658 if (ret)
2659 goto out;
2660 }
2661 start = skb_checksum_start_offset(skb);
2662 offset = start + offsetof(struct sctphdr, checksum);
2663 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2664 ret = -EINVAL;
2665 goto out;
2666 }
2667 if (skb_cloned(skb) &&
2668 !skb_clone_writable(skb, offset + sizeof(__le32))) {
2669 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2670 if (ret)
2671 goto out;
2672 }
2673 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2674 skb->len - start, ~(__u32)0,
2675 crc32c_csum_stub));
2676 *(__le32 *)(skb->data + offset) = crc32c_csum;
2677 skb->ip_summed = CHECKSUM_NONE;
2678 skb->csum_not_inet = 0;
2679out:
2680 return ret;
2681}
2682
2614__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2615{
2616 __be16 type = skb->protocol;
2617
2618 /* Tunnel gso handlers can set protocol to ethernet. */
2619 if (type == htons(ETH_P_TEB)) {
2620 struct ethhdr *eth;
2621

--- 326 unchanged lines hidden (view full) ---

2948 netdev_features_t features)
2949{
2950 if (skb_vlan_tag_present(skb) &&
2951 !vlan_hw_offload_capable(features, skb->vlan_proto))
2952 skb = __vlan_hwaccel_push_inside(skb);
2953 return skb;
2954}
2955
2683__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2684{
2685 __be16 type = skb->protocol;
2686
2687 /* Tunnel gso handlers can set protocol to ethernet. */
2688 if (type == htons(ETH_P_TEB)) {
2689 struct ethhdr *eth;
2690

--- 326 unchanged lines hidden (view full) ---

3017 netdev_features_t features)
3018{
3019 if (skb_vlan_tag_present(skb) &&
3020 !vlan_hw_offload_capable(features, skb->vlan_proto))
3021 skb = __vlan_hwaccel_push_inside(skb);
3022 return skb;
3023}
3024
3025int skb_csum_hwoffload_help(struct sk_buff *skb,
3026 const netdev_features_t features)
3027{
3028 if (unlikely(skb->csum_not_inet))
3029 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3030 skb_crc32c_csum_help(skb);
3031
3032 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3033}
3034EXPORT_SYMBOL(skb_csum_hwoffload_help);
3035
2956static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2957{
2958 netdev_features_t features;
2959
2960 features = netif_skb_features(skb);
2961 skb = validate_xmit_vlan(skb, features);
2962 if (unlikely(!skb))
2963 goto out_null;

--- 22 unchanged lines hidden (view full) ---

2986 */
2987 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2988 if (skb->encapsulation)
2989 skb_set_inner_transport_header(skb,
2990 skb_checksum_start_offset(skb));
2991 else
2992 skb_set_transport_header(skb,
2993 skb_checksum_start_offset(skb));
3036static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
3037{
3038 netdev_features_t features;
3039
3040 features = netif_skb_features(skb);
3041 skb = validate_xmit_vlan(skb, features);
3042 if (unlikely(!skb))
3043 goto out_null;

--- 22 unchanged lines hidden (view full) ---

3066 */
3067 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3068 if (skb->encapsulation)
3069 skb_set_inner_transport_header(skb,
3070 skb_checksum_start_offset(skb));
3071 else
3072 skb_set_transport_header(skb,
3073 skb_checksum_start_offset(skb));
2994 if (!(features & NETIF_F_CSUM_MASK) &&
2995 skb_checksum_help(skb))
3074 if (skb_csum_hwoffload_help(skb, features))
2996 goto out_kfree_skb;
2997 }
2998 }
2999
3000 return skb;
3001
3002out_kfree_skb:
3003 kfree_skb(skb);

--- 169 unchanged lines hidden (view full) ---

3173 struct tcf_result cl_res;
3174
3175 if (!cl)
3176 return skb;
3177
3178 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3179 qdisc_bstats_cpu_update(cl->q, skb);
3180
3075 goto out_kfree_skb;
3076 }
3077 }
3078
3079 return skb;
3080
3081out_kfree_skb:
3082 kfree_skb(skb);

--- 169 unchanged lines hidden (view full) ---

3252 struct tcf_result cl_res;
3253
3254 if (!cl)
3255 return skb;
3256
3257 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3258 qdisc_bstats_cpu_update(cl->q, skb);
3259
3181 switch (tc_classify(skb, cl, &cl_res, false)) {
3260 switch (tcf_classify(skb, cl, &cl_res, false)) {
3182 case TC_ACT_OK:
3183 case TC_ACT_RECLASSIFY:
3184 skb->tc_index = TC_H_MIN(cl_res.classid);
3185 break;
3186 case TC_ACT_SHOT:
3187 qdisc_qstats_cpu_drop(cl->q);
3188 *ret = NET_XMIT_DROP;
3189 kfree_skb(skb);

--- 753 unchanged lines hidden (view full) ---

3943 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3944 *pt_prev = NULL;
3945 }
3946
3947 qdisc_skb_cb(skb)->pkt_len = skb->len;
3948 skb->tc_at_ingress = 1;
3949 qdisc_bstats_cpu_update(cl->q, skb);
3950
3261 case TC_ACT_OK:
3262 case TC_ACT_RECLASSIFY:
3263 skb->tc_index = TC_H_MIN(cl_res.classid);
3264 break;
3265 case TC_ACT_SHOT:
3266 qdisc_qstats_cpu_drop(cl->q);
3267 *ret = NET_XMIT_DROP;
3268 kfree_skb(skb);

--- 753 unchanged lines hidden (view full) ---

4022 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4023 *pt_prev = NULL;
4024 }
4025
4026 qdisc_skb_cb(skb)->pkt_len = skb->len;
4027 skb->tc_at_ingress = 1;
4028 qdisc_bstats_cpu_update(cl->q, skb);
4029
3951 switch (tc_classify(skb, cl, &cl_res, false)) {
4030 switch (tcf_classify(skb, cl, &cl_res, false)) {
3952 case TC_ACT_OK:
3953 case TC_ACT_RECLASSIFY:
3954 skb->tc_index = TC_H_MIN(cl_res.classid);
3955 break;
3956 case TC_ACT_SHOT:
3957 qdisc_qstats_cpu_drop(cl->q);
3958 kfree_skb(skb);
3959 return NULL;

--- 671 unchanged lines hidden (view full) ---

4631 struct list_head *head = &offload_base;
4632 int same_flow;
4633 enum gro_result ret;
4634 int grow;
4635
4636 if (netif_elide_gro(skb->dev))
4637 goto normal;
4638
4031 case TC_ACT_OK:
4032 case TC_ACT_RECLASSIFY:
4033 skb->tc_index = TC_H_MIN(cl_res.classid);
4034 break;
4035 case TC_ACT_SHOT:
4036 qdisc_qstats_cpu_drop(cl->q);
4037 kfree_skb(skb);
4038 return NULL;

--- 671 unchanged lines hidden (view full) ---

4710 struct list_head *head = &offload_base;
4711 int same_flow;
4712 enum gro_result ret;
4713 int grow;
4714
4715 if (netif_elide_gro(skb->dev))
4716 goto normal;
4717
4639 if (skb->csum_bad)
4640 goto normal;
4641
4642 gro_list_prepare(napi, skb);
4643
4644 rcu_read_lock();
4645 list_for_each_entry_rcu(ptype, head, list) {
4646 if (ptype->type != type || !ptype->callbacks.gro_receive)
4647 continue;
4648
4649 skb_set_network_header(skb, skb_gro_offset(skb));

--- 2353 unchanged lines hidden (view full) ---

7003
7004 /* Notify protocols, that we are about to destroy
7005 * this device. They should clean all the things.
7006 */
7007 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7008
7009 if (!dev->rtnl_link_ops ||
7010 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
4718 gro_list_prepare(napi, skb);
4719
4720 rcu_read_lock();
4721 list_for_each_entry_rcu(ptype, head, list) {
4722 if (ptype->type != type || !ptype->callbacks.gro_receive)
4723 continue;
4724
4725 skb_set_network_header(skb, skb_gro_offset(skb));

--- 2353 unchanged lines hidden (view full) ---

7079
7080 /* Notify protocols, that we are about to destroy
7081 * this device. They should clean all the things.
7082 */
7083 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7084
7085 if (!dev->rtnl_link_ops ||
7086 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7011 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
7087 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
7012 GFP_KERNEL);
7013
7014 /*
7015 * Flush the unicast and multicast chains
7016 */
7017 dev_uc_flush(dev);
7018 dev_mc_flush(dev);
7019

--- 1572 unchanged lines hidden --- 		7088 GFP_KERNEL);
7089
7090 /*
7091 * Flush the unicast and multicast chains
7092 */
7093 dev_uc_flush(dev);
7094 dev_mc_flush(dev);
7095

--- 1572 unchanged lines hidden ---