inet_hashtables.c (2a906db2824b75444982f5e9df870106982afca8) inet_hashtables.c (08eaef90403110e51861d93e8008a355af467bbe)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Generic INET transport hashtables
8 *

--- 78 unchanged lines hidden (view full) ---

87void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
88{
89 if (hlist_empty(&tb->owners)) {
90 __hlist_del(&tb->node);
91 kmem_cache_free(cachep, tb);
92 }
93}
94
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Generic INET transport hashtables
8 *

--- 78 unchanged lines hidden (view full) ---

87void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
88{
89 if (hlist_empty(&tb->owners)) {
90 __hlist_del(&tb->node);
91 kmem_cache_free(cachep, tb);
92 }
93}
94
95bool inet_bind_bucket_match(const struct inet_bind_bucket *tb, const struct net *net,
96 unsigned short port, int l3mdev)
97{
98 return net_eq(ib_net(tb), net) && tb->port == port &&
99 tb->l3mdev == l3mdev;
100}
101
102static void inet_bind2_bucket_init(struct inet_bind2_bucket *tb,
103 struct net *net,
104 struct inet_bind_hashbucket *head,
105 unsigned short port, int l3mdev,
106 const struct sock *sk)
107{
108 write_pnet(&tb->ib_net, net);
109 tb->l3mdev = l3mdev;
110 tb->port = port;
111#if IS_ENABLED(CONFIG_IPV6)
112 if (sk->sk_family == AF_INET6)
113 tb->v6_rcv_saddr = sk->sk_v6_rcv_saddr;
114 else
115#endif
116 tb->rcv_saddr = sk->sk_rcv_saddr;
117 INIT_HLIST_HEAD(&tb->owners);
118 hlist_add_head(&tb->node, &head->chain);
119}
120
121struct inet_bind2_bucket *inet_bind2_bucket_create(struct kmem_cache *cachep,
122 struct net *net,
123 struct inet_bind_hashbucket *head,
124 unsigned short port,
125 int l3mdev,
126 const struct sock *sk)
127{
128 struct inet_bind2_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
129
130 if (tb)
131 inet_bind2_bucket_init(tb, net, head, port, l3mdev, sk);
132
133 return tb;
134}
135
136/* Caller must hold hashbucket lock for this tb with local BH disabled */
137void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb)
138{
139 if (hlist_empty(&tb->owners)) {
140 __hlist_del(&tb->node);
141 kmem_cache_free(cachep, tb);
142 }
143}
144
145static bool inet_bind2_bucket_addr_match(const struct inet_bind2_bucket *tb2,
146 const struct sock *sk)
147{
148#if IS_ENABLED(CONFIG_IPV6)
149 if (sk->sk_family == AF_INET6)
150 return ipv6_addr_equal(&tb2->v6_rcv_saddr,
151 &sk->sk_v6_rcv_saddr);
152#endif
153 return tb2->rcv_saddr == sk->sk_rcv_saddr;
154}
155
95void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
156void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
96 const unsigned short snum)
157 struct inet_bind2_bucket *tb2, unsigned short port)
97{
158{
98 inet_sk(sk)->inet_num = snum;
159 inet_sk(sk)->inet_num = port;
99 sk_add_bind_node(sk, &tb->owners);
100 inet_csk(sk)->icsk_bind_hash = tb;
160 sk_add_bind_node(sk, &tb->owners);
161 inet_csk(sk)->icsk_bind_hash = tb;
162 sk_add_bind2_node(sk, &tb2->owners);
163 inet_csk(sk)->icsk_bind2_hash = tb2;
101}
102
103/*
104 * Get rid of any references to a local port held by the given sock.
105 */
106static void __inet_put_port(struct sock *sk)
107{
108 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
164}
165
166/*
167 * Get rid of any references to a local port held by the given sock.
168 */
169static void __inet_put_port(struct sock *sk)
170{
171 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
109 const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
110 hashinfo->bhash_size);
111 struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
172 struct inet_bind_hashbucket *head, *head2;
173 struct net *net = sock_net(sk);
112 struct inet_bind_bucket *tb;
174 struct inet_bind_bucket *tb;
175 int bhash;
113
176
177 bhash = inet_bhashfn(net, inet_sk(sk)->inet_num, hashinfo->bhash_size);
178 head = &hashinfo->bhash[bhash];
179 head2 = inet_bhashfn_portaddr(hashinfo, sk, net, inet_sk(sk)->inet_num);
180
114 spin_lock(&head->lock);
115 tb = inet_csk(sk)->icsk_bind_hash;
116 __sk_del_bind_node(sk);
117 inet_csk(sk)->icsk_bind_hash = NULL;
118 inet_sk(sk)->inet_num = 0;
119 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
181 spin_lock(&head->lock);
182 tb = inet_csk(sk)->icsk_bind_hash;
183 __sk_del_bind_node(sk);
184 inet_csk(sk)->icsk_bind_hash = NULL;
185 inet_sk(sk)->inet_num = 0;
186 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
187
188 spin_lock(&head2->lock);
189 if (inet_csk(sk)->icsk_bind2_hash) {
190 struct inet_bind2_bucket *tb2 = inet_csk(sk)->icsk_bind2_hash;
191
192 __sk_del_bind2_node(sk);
193 inet_csk(sk)->icsk_bind2_hash = NULL;
194 inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2);
195 }
196 spin_unlock(&head2->lock);
197
120 spin_unlock(&head->lock);
121}
122
123void inet_put_port(struct sock *sk)
124{
125 local_bh_disable();
126 __inet_put_port(sk);
127 local_bh_enable();
128}
129EXPORT_SYMBOL(inet_put_port);
130
131int __inet_inherit_port(const struct sock *sk, struct sock *child)
132{
133 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
134 unsigned short port = inet_sk(child)->inet_num;
198 spin_unlock(&head->lock);
199}
200
201void inet_put_port(struct sock *sk)
202{
203 local_bh_disable();
204 __inet_put_port(sk);
205 local_bh_enable();
206}
207EXPORT_SYMBOL(inet_put_port);
208
209int __inet_inherit_port(const struct sock *sk, struct sock *child)
210{
211 struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
212 unsigned short port = inet_sk(child)->inet_num;
135 const int bhash = inet_bhashfn(sock_net(sk), port,
136 table->bhash_size);
137 struct inet_bind_hashbucket *head = &table->bhash[bhash];
213 struct inet_bind_hashbucket *head, *head2;
214 bool created_inet_bind_bucket = false;
215 struct net *net = sock_net(sk);
216 bool update_fastreuse = false;
217 struct inet_bind2_bucket *tb2;
138 struct inet_bind_bucket *tb;
218 struct inet_bind_bucket *tb;
139 int l3mdev;
219 int bhash, l3mdev;
140
220
221 bhash = inet_bhashfn(net, port, table->bhash_size);
222 head = &table->bhash[bhash];
223 head2 = inet_bhashfn_portaddr(table, child, net, port);
224
141 spin_lock(&head->lock);
225 spin_lock(&head->lock);
226 spin_lock(&head2->lock);
142 tb = inet_csk(sk)->icsk_bind_hash;
227 tb = inet_csk(sk)->icsk_bind_hash;
143 if (unlikely(!tb)) {
228 tb2 = inet_csk(sk)->icsk_bind2_hash;
229 if (unlikely(!tb || !tb2)) {
230 spin_unlock(&head2->lock);
144 spin_unlock(&head->lock);
145 return -ENOENT;
146 }
147 if (tb->port != port) {
148 l3mdev = inet_sk_bound_l3mdev(sk);
149
150 /* NOTE: using tproxy and redirecting skbs to a proxy
151 * on a different listener port breaks the assumption
152 * that the listener socket's icsk_bind_hash is the same
153 * as that of the child socket. We have to look up or
154 * create a new bind bucket for the child here. */
155 inet_bind_bucket_for_each(tb, &head->chain) {
231 spin_unlock(&head->lock);
232 return -ENOENT;
233 }
234 if (tb->port != port) {
235 l3mdev = inet_sk_bound_l3mdev(sk);
236
237 /* NOTE: using tproxy and redirecting skbs to a proxy
238 * on a different listener port breaks the assumption
239 * that the listener socket's icsk_bind_hash is the same
240 * as that of the child socket. We have to look up or
241 * create a new bind bucket for the child here. */
242 inet_bind_bucket_for_each(tb, &head->chain) {
156 if (net_eq(ib_net(tb), sock_net(sk)) &&
157 tb->l3mdev == l3mdev && tb->port == port)
243 if (inet_bind_bucket_match(tb, net, port, l3mdev))
158 break;
159 }
160 if (!tb) {
161 tb = inet_bind_bucket_create(table->bind_bucket_cachep,
244 break;
245 }
246 if (!tb) {
247 tb = inet_bind_bucket_create(table->bind_bucket_cachep,
162 sock_net(sk), head, port,
163 l3mdev);
248 net, head, port, l3mdev);
164 if (!tb) {
249 if (!tb) {
250 spin_unlock(&head2->lock);
165 spin_unlock(&head->lock);
166 return -ENOMEM;
167 }
251 spin_unlock(&head->lock);
252 return -ENOMEM;
253 }
254 created_inet_bind_bucket = true;
168 }
255 }
169 inet_csk_update_fastreuse(tb, child);
256 update_fastreuse = true;
257
258 goto bhash2_find;
259 } else if (!inet_bind2_bucket_addr_match(tb2, child)) {
260 l3mdev = inet_sk_bound_l3mdev(sk);
261
262bhash2_find:
263 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, child);
264 if (!tb2) {
265 tb2 = inet_bind2_bucket_create(table->bind2_bucket_cachep,
266 net, head2, port,
267 l3mdev, child);
268 if (!tb2)
269 goto error;
270 }
170 }
271 }
171 inet_bind_hash(child, tb, port);
272 if (update_fastreuse)
273 inet_csk_update_fastreuse(tb, child);
274 inet_bind_hash(child, tb, tb2, port);
275 spin_unlock(&head2->lock);
172 spin_unlock(&head->lock);
173
174 return 0;
276 spin_unlock(&head->lock);
277
278 return 0;
279
280error:
281 if (created_inet_bind_bucket)
282 inet_bind_bucket_destroy(table->bind_bucket_cachep, tb);
283 spin_unlock(&head2->lock);
284 spin_unlock(&head->lock);
285 return -ENOMEM;
175}
176EXPORT_SYMBOL_GPL(__inet_inherit_port);
177
178static struct inet_listen_hashbucket *
179inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk)
180{
181 u32 hash;
182

--- 331 unchanged lines hidden (view full) ---

514/* Insert a socket into ehash, and eventually remove another one
515 * (The another one can be a SYN_RECV or TIMEWAIT)
516 * If an existing socket already exists, socket sk is not inserted,
517 * and sets found_dup_sk parameter to true.
518 */
519bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk)
520{
521 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
286}
287EXPORT_SYMBOL_GPL(__inet_inherit_port);
288
289static struct inet_listen_hashbucket *
290inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk)
291{
292 u32 hash;
293

--- 331 unchanged lines hidden (view full) ---

625/* Insert a socket into ehash, and eventually remove another one
626 * (The another one can be a SYN_RECV or TIMEWAIT)
627 * If an existing socket already exists, socket sk is not inserted,
628 * and sets found_dup_sk parameter to true.
629 */
630bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk)
631{
632 struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
522 struct hlist_nulls_head *list;
523 struct inet_ehash_bucket *head;
633 struct inet_ehash_bucket *head;
634 struct hlist_nulls_head *list;
524 spinlock_t *lock;
525 bool ret = true;
526
527 WARN_ON_ONCE(!sk_unhashed(sk));
528
529 sk->sk_hash = sk_ehashfn(sk);
530 head = inet_ehash_bucket(hashinfo, sk->sk_hash);
531 list = &head->chain;

--- 138 unchanged lines hidden (view full) ---

670 }
671 __sk_nulls_del_node_init_rcu(sk);
672 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
673 spin_unlock_bh(lock);
674 }
675}
676EXPORT_SYMBOL_GPL(inet_unhash);
677
635 spinlock_t *lock;
636 bool ret = true;
637
638 WARN_ON_ONCE(!sk_unhashed(sk));
639
640 sk->sk_hash = sk_ehashfn(sk);
641 head = inet_ehash_bucket(hashinfo, sk->sk_hash);
642 list = &head->chain;

--- 138 unchanged lines hidden (view full) ---

781 }
782 __sk_nulls_del_node_init_rcu(sk);
783 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
784 spin_unlock_bh(lock);
785 }
786}
787EXPORT_SYMBOL_GPL(inet_unhash);
788
789static bool inet_bind2_bucket_match(const struct inet_bind2_bucket *tb,
790 const struct net *net, unsigned short port,
791 int l3mdev, const struct sock *sk)
792{
793#if IS_ENABLED(CONFIG_IPV6)
794 if (sk->sk_family == AF_INET6)
795 return net_eq(ib2_net(tb), net) && tb->port == port &&
796 tb->l3mdev == l3mdev &&
797 ipv6_addr_equal(&tb->v6_rcv_saddr, &sk->sk_v6_rcv_saddr);
798 else
799#endif
800 return net_eq(ib2_net(tb), net) && tb->port == port &&
801 tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr;
802}
803
804bool inet_bind2_bucket_match_addr_any(const struct inet_bind2_bucket *tb, const struct net *net,
805 unsigned short port, int l3mdev, const struct sock *sk)
806{
807#if IS_ENABLED(CONFIG_IPV6)
808 struct in6_addr addr_any = {};
809
810 if (sk->sk_family == AF_INET6)
811 return net_eq(ib2_net(tb), net) && tb->port == port &&
812 tb->l3mdev == l3mdev &&
813 ipv6_addr_equal(&tb->v6_rcv_saddr, &addr_any);
814 else
815#endif
816 return net_eq(ib2_net(tb), net) && tb->port == port &&
817 tb->l3mdev == l3mdev && tb->rcv_saddr == 0;
818}
819
820/* The socket's bhash2 hashbucket spinlock must be held when this is called */
821struct inet_bind2_bucket *
822inet_bind2_bucket_find(const struct inet_bind_hashbucket *head, const struct net *net,
823 unsigned short port, int l3mdev, const struct sock *sk)
824{
825 struct inet_bind2_bucket *bhash2 = NULL;
826
827 inet_bind_bucket_for_each(bhash2, &head->chain)
828 if (inet_bind2_bucket_match(bhash2, net, port, l3mdev, sk))
829 break;
830
831 return bhash2;
832}
833
834struct inet_bind_hashbucket *
835inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port)
836{
837 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
838 u32 hash;
839#if IS_ENABLED(CONFIG_IPV6)
840 struct in6_addr addr_any = {};
841
842 if (sk->sk_family == AF_INET6)
843 hash = ipv6_portaddr_hash(net, &addr_any, port);
844 else
845#endif
846 hash = ipv4_portaddr_hash(net, 0, port);
847
848 return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)];
849}
850
851int inet_bhash2_update_saddr(struct inet_bind_hashbucket *prev_saddr, struct sock *sk)
852{
853 struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
854 struct inet_bind2_bucket *tb2, *new_tb2;
855 int l3mdev = inet_sk_bound_l3mdev(sk);
856 struct inet_bind_hashbucket *head2;
857 int port = inet_sk(sk)->inet_num;
858 struct net *net = sock_net(sk);
859
860 /* Allocate a bind2 bucket ahead of time to avoid permanently putting
861 * the bhash2 table in an inconsistent state if a new tb2 bucket
862 * allocation fails.
863 */
864 new_tb2 = kmem_cache_alloc(hinfo->bind2_bucket_cachep, GFP_ATOMIC);
865 if (!new_tb2)
866 return -ENOMEM;
867
868 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port);
869
870 if (prev_saddr) {
871 spin_lock_bh(&prev_saddr->lock);
872 __sk_del_bind2_node(sk);
873 inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep,
874 inet_csk(sk)->icsk_bind2_hash);
875 spin_unlock_bh(&prev_saddr->lock);
876 }
877
878 spin_lock_bh(&head2->lock);
879 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk);
880 if (!tb2) {
881 tb2 = new_tb2;
882 inet_bind2_bucket_init(tb2, net, head2, port, l3mdev, sk);
883 }
884 sk_add_bind2_node(sk, &tb2->owners);
885 inet_csk(sk)->icsk_bind2_hash = tb2;
886 spin_unlock_bh(&head2->lock);
887
888 if (tb2 != new_tb2)
889 kmem_cache_free(hinfo->bind2_bucket_cachep, new_tb2);
890
891 return 0;
892}
893EXPORT_SYMBOL_GPL(inet_bhash2_update_saddr);
894
678/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm
679 * Note that we use 32bit integers (vs RFC 'short integers')
680 * because 2^16 is not a multiple of num_ephemeral and this
681 * property might be used by clever attacker.
682 * RFC claims using TABLE_LENGTH=10 buckets gives an improvement, though
683 * attacks were since demonstrated, thus we use 65536 instead to really
684 * give more isolation and privacy, at the expense of 256kB of kernel
685 * memory.
686 */
687#define INET_TABLE_PERTURB_SHIFT 16
688#define INET_TABLE_PERTURB_SIZE (1 << INET_TABLE_PERTURB_SHIFT)
689static u32 *table_perturb;
690
691int __inet_hash_connect(struct inet_timewait_death_row *death_row,
692 struct sock *sk, u64 port_offset,
693 int (*check_established)(struct inet_timewait_death_row *,
694 struct sock *, __u16, struct inet_timewait_sock **))
695{
696 struct inet_hashinfo *hinfo = death_row->hashinfo;
895/* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm
896 * Note that we use 32bit integers (vs RFC 'short integers')
897 * because 2^16 is not a multiple of num_ephemeral and this
898 * property might be used by clever attacker.
899 * RFC claims using TABLE_LENGTH=10 buckets gives an improvement, though
900 * attacks were since demonstrated, thus we use 65536 instead to really
901 * give more isolation and privacy, at the expense of 256kB of kernel
902 * memory.
903 */
904#define INET_TABLE_PERTURB_SHIFT 16
905#define INET_TABLE_PERTURB_SIZE (1 << INET_TABLE_PERTURB_SHIFT)
906static u32 *table_perturb;
907
908int __inet_hash_connect(struct inet_timewait_death_row *death_row,
909 struct sock *sk, u64 port_offset,
910 int (*check_established)(struct inet_timewait_death_row *,
911 struct sock *, __u16, struct inet_timewait_sock **))
912{
913 struct inet_hashinfo *hinfo = death_row->hashinfo;
914 struct inet_bind_hashbucket *head, *head2;
697 struct inet_timewait_sock *tw = NULL;
915 struct inet_timewait_sock *tw = NULL;
698 struct inet_bind_hashbucket *head;
699 int port = inet_sk(sk)->inet_num;
700 struct net *net = sock_net(sk);
916 int port = inet_sk(sk)->inet_num;
917 struct net *net = sock_net(sk);
918 struct inet_bind2_bucket *tb2;
701 struct inet_bind_bucket *tb;
919 struct inet_bind_bucket *tb;
920 bool tb_created = false;
702 u32 remaining, offset;
703 int ret, i, low, high;
704 int l3mdev;
705 u32 index;
706
707 if (port) {
708 head = &hinfo->bhash[inet_bhashfn(net, port,
709 hinfo->bhash_size)];

--- 40 unchanged lines hidden (view full) ---

750 head = &hinfo->bhash[inet_bhashfn(net, port,
751 hinfo->bhash_size)];
752 spin_lock_bh(&head->lock);
753
754 /* Does not bother with rcv_saddr checks, because
755 * the established check is already unique enough.
756 */
757 inet_bind_bucket_for_each(tb, &head->chain) {
921 u32 remaining, offset;
922 int ret, i, low, high;
923 int l3mdev;
924 u32 index;
925
926 if (port) {
927 head = &hinfo->bhash[inet_bhashfn(net, port,
928 hinfo->bhash_size)];

--- 40 unchanged lines hidden (view full) ---

969 head = &hinfo->bhash[inet_bhashfn(net, port,
970 hinfo->bhash_size)];
971 spin_lock_bh(&head->lock);
972
973 /* Does not bother with rcv_saddr checks, because
974 * the established check is already unique enough.
975 */
976 inet_bind_bucket_for_each(tb, &head->chain) {
758 if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
759 tb->port == port) {
977 if (inet_bind_bucket_match(tb, net, port, l3mdev)) {
760 if (tb->fastreuse >= 0 ||
761 tb->fastreuseport >= 0)
762 goto next_port;
763 WARN_ON(hlist_empty(&tb->owners));
764 if (!check_established(death_row, sk,
765 port, &tw))
766 goto ok;
767 goto next_port;
768 }
769 }
770
771 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
772 net, head, port, l3mdev);
773 if (!tb) {
774 spin_unlock_bh(&head->lock);
775 return -ENOMEM;
776 }
978 if (tb->fastreuse >= 0 ||
979 tb->fastreuseport >= 0)
980 goto next_port;
981 WARN_ON(hlist_empty(&tb->owners));
982 if (!check_established(death_row, sk,
983 port, &tw))
984 goto ok;
985 goto next_port;
986 }
987 }
988
989 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
990 net, head, port, l3mdev);
991 if (!tb) {
992 spin_unlock_bh(&head->lock);
993 return -ENOMEM;
994 }
995 tb_created = true;
777 tb->fastreuse = -1;
778 tb->fastreuseport = -1;
779 goto ok;
780next_port:
781 spin_unlock_bh(&head->lock);
782 cond_resched();
783 }
784
785 offset++;
786 if ((offset & 1) && remaining > 1)
787 goto other_parity_scan;
788
789 return -EADDRNOTAVAIL;
790
791ok:
996 tb->fastreuse = -1;
997 tb->fastreuseport = -1;
998 goto ok;
999next_port:
1000 spin_unlock_bh(&head->lock);
1001 cond_resched();
1002 }
1003
1004 offset++;
1005 if ((offset & 1) && remaining > 1)
1006 goto other_parity_scan;
1007
1008 return -EADDRNOTAVAIL;
1009
1010ok:
1011 /* Find the corresponding tb2 bucket since we need to
1012 * add the socket to the bhash2 table as well
1013 */
1014 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port);
1015 spin_lock(&head2->lock);
1016
1017 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk);
1018 if (!tb2) {
1019 tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, net,
1020 head2, port, l3mdev, sk);
1021 if (!tb2)
1022 goto error;
1023 }
1024
792 /* Here we want to add a little bit of randomness to the next source
793 * port that will be chosen. We use a max() with a random here so that
794 * on low contention the randomness is maximal and on high contention
795 * it may be inexistent.
796 */
797 i = max_t(int, i, (prandom_u32() & 7) * 2);
798 WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
799
800 /* Head lock still held and bh's disabled */
1025 /* Here we want to add a little bit of randomness to the next source
1026 * port that will be chosen. We use a max() with a random here so that
1027 * on low contention the randomness is maximal and on high contention
1028 * it may be inexistent.
1029 */
1030 i = max_t(int, i, (prandom_u32() & 7) * 2);
1031 WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
1032
1033 /* Head lock still held and bh's disabled */
801 inet_bind_hash(sk, tb, port);
1034 inet_bind_hash(sk, tb, tb2, port);
1035
1036 spin_unlock(&head2->lock);
1037
802 if (sk_unhashed(sk)) {
803 inet_sk(sk)->inet_sport = htons(port);
804 inet_ehash_nolisten(sk, (struct sock *)tw, NULL);
805 }
806 if (tw)
807 inet_twsk_bind_unhash(tw, hinfo);
808 spin_unlock(&head->lock);
809 if (tw)
810 inet_twsk_deschedule_put(tw);
811 local_bh_enable();
812 return 0;
1038 if (sk_unhashed(sk)) {
1039 inet_sk(sk)->inet_sport = htons(port);
1040 inet_ehash_nolisten(sk, (struct sock *)tw, NULL);
1041 }
1042 if (tw)
1043 inet_twsk_bind_unhash(tw, hinfo);
1044 spin_unlock(&head->lock);
1045 if (tw)
1046 inet_twsk_deschedule_put(tw);
1047 local_bh_enable();
1048 return 0;
1049
1050error:
1051 spin_unlock(&head2->lock);
1052 if (tb_created)
1053 inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb);
1054 spin_unlock_bh(&head->lock);
1055 return -ENOMEM;
813}
814
815/*
816 * Bind a port for a connect operation and hash it.
817 */
818int inet_hash_connect(struct inet_timewait_death_row *death_row,
819 struct sock *sk)
820{

--- 84 unchanged lines hidden --- 		1056}
1057
1058/*
1059 * Bind a port for a connect operation and hash it.
1060 */
1061int inet_hash_connect(struct inet_timewait_death_row *death_row,
1062 struct sock *sk)
1063{

--- 84 unchanged lines hidden ---