1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Generic socket support routines. Memory allocators, socket lock/release 7 * handler for protocols to use and generic option handler. 8 * 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 * 85 * 86 * This program is free software; you can redistribute it and/or 87 * modify it under the terms of the GNU General Public License 88 * as published by the Free Software Foundation; either version 89 * 2 of the License, or (at your option) any later version. 90 */ 91 92 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 93 94 #include <linux/capability.h> 95 #include <linux/errno.h> 96 #include <linux/errqueue.h> 97 #include <linux/types.h> 98 #include <linux/socket.h> 99 #include <linux/in.h> 100 #include <linux/kernel.h> 101 #include <linux/module.h> 102 #include <linux/proc_fs.h> 103 #include <linux/seq_file.h> 104 #include <linux/sched.h> 105 #include <linux/sched/mm.h> 106 #include <linux/timer.h> 107 #include <linux/string.h> 108 #include <linux/sockios.h> 109 #include <linux/net.h> 110 #include <linux/mm.h> 111 #include <linux/slab.h> 112 #include <linux/interrupt.h> 113 #include <linux/poll.h> 114 #include <linux/tcp.h> 115 #include <linux/init.h> 116 #include <linux/highmem.h> 117 #include <linux/user_namespace.h> 118 #include <linux/static_key.h> 119 #include <linux/memcontrol.h> 120 #include <linux/prefetch.h> 121 122 #include <linux/uaccess.h> 123 124 #include <linux/netdevice.h> 125 #include <net/protocol.h> 126 #include <linux/skbuff.h> 127 #include <net/net_namespace.h> 128 #include <net/request_sock.h> 129 #include <net/sock.h> 130 #include <linux/net_tstamp.h> 131 #include <net/xfrm.h> 132 #include <linux/ipsec.h> 133 #include <net/cls_cgroup.h> 134 #include <net/netprio_cgroup.h> 135 #include <linux/sock_diag.h> 136 137 #include <linux/filter.h> 138 #include <net/sock_reuseport.h> 139 140 #include <trace/events/sock.h> 141 142 #include <net/tcp.h> 143 #include <net/busy_poll.h> 144 145 static DEFINE_MUTEX(proto_list_mutex); 146 static LIST_HEAD(proto_list); 147 148 static void sock_inuse_add(struct net *net, int val); 149 150 /** 151 * sk_ns_capable - General socket capability test 152 * @sk: Socket to use a capability on or through 153 * @user_ns: The user namespace of the capability to use 154 * @cap: The capability to use 155 * 156 * Test to see if the opener of the socket had when the socket was 157 * created and the current process has the capability @cap in the user 158 * namespace @user_ns. 159 */ 160 bool sk_ns_capable(const struct sock *sk, 161 struct user_namespace *user_ns, int cap) 162 { 163 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 164 ns_capable(user_ns, cap); 165 } 166 EXPORT_SYMBOL(sk_ns_capable); 167 168 /** 169 * sk_capable - Socket global capability test 170 * @sk: Socket to use a capability on or through 171 * @cap: The global capability to use 172 * 173 * Test to see if the opener of the socket had when the socket was 174 * created and the current process has the capability @cap in all user 175 * namespaces. 176 */ 177 bool sk_capable(const struct sock *sk, int cap) 178 { 179 return sk_ns_capable(sk, &init_user_ns, cap); 180 } 181 EXPORT_SYMBOL(sk_capable); 182 183 /** 184 * sk_net_capable - Network namespace socket capability test 185 * @sk: Socket to use a capability on or through 186 * @cap: The capability to use 187 * 188 * Test to see if the opener of the socket had when the socket was created 189 * and the current process has the capability @cap over the network namespace 190 * the socket is a member of. 191 */ 192 bool sk_net_capable(const struct sock *sk, int cap) 193 { 194 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 195 } 196 EXPORT_SYMBOL(sk_net_capable); 197 198 /* 199 * Each address family might have different locking rules, so we have 200 * one slock key per address family and separate keys for internal and 201 * userspace sockets. 202 */ 203 static struct lock_class_key af_family_keys[AF_MAX]; 204 static struct lock_class_key af_family_kern_keys[AF_MAX]; 205 static struct lock_class_key af_family_slock_keys[AF_MAX]; 206 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 207 208 /* 209 * Make lock validator output more readable. (we pre-construct these 210 * strings build-time, so that runtime initialization of socket 211 * locks is fast): 212 */ 213 214 #define _sock_locks(x) \ 215 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 216 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 217 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 218 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 219 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 220 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 221 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 222 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 223 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 224 x "27" , x "28" , x "AF_CAN" , \ 225 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 226 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 227 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 228 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 229 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 230 x "AF_MAX" 231 232 static const char *const af_family_key_strings[AF_MAX+1] = { 233 _sock_locks("sk_lock-") 234 }; 235 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 236 _sock_locks("slock-") 237 }; 238 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 239 _sock_locks("clock-") 240 }; 241 242 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 243 _sock_locks("k-sk_lock-") 244 }; 245 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 246 _sock_locks("k-slock-") 247 }; 248 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 249 _sock_locks("k-clock-") 250 }; 251 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 252 "rlock-AF_UNSPEC", "rlock-AF_UNIX" , "rlock-AF_INET" , 253 "rlock-AF_AX25" , "rlock-AF_IPX" , "rlock-AF_APPLETALK", 254 "rlock-AF_NETROM", "rlock-AF_BRIDGE" , "rlock-AF_ATMPVC" , 255 "rlock-AF_X25" , "rlock-AF_INET6" , "rlock-AF_ROSE" , 256 "rlock-AF_DECnet", "rlock-AF_NETBEUI" , "rlock-AF_SECURITY" , 257 "rlock-AF_KEY" , "rlock-AF_NETLINK" , "rlock-AF_PACKET" , 258 "rlock-AF_ASH" , "rlock-AF_ECONET" , "rlock-AF_ATMSVC" , 259 "rlock-AF_RDS" , "rlock-AF_SNA" , "rlock-AF_IRDA" , 260 "rlock-AF_PPPOX" , "rlock-AF_WANPIPE" , "rlock-AF_LLC" , 261 "rlock-27" , "rlock-28" , "rlock-AF_CAN" , 262 "rlock-AF_TIPC" , "rlock-AF_BLUETOOTH", "rlock-AF_IUCV" , 263 "rlock-AF_RXRPC" , "rlock-AF_ISDN" , "rlock-AF_PHONET" , 264 "rlock-AF_IEEE802154", "rlock-AF_CAIF" , "rlock-AF_ALG" , 265 "rlock-AF_NFC" , "rlock-AF_VSOCK" , "rlock-AF_KCM" , 266 "rlock-AF_QIPCRTR", "rlock-AF_SMC" , "rlock-AF_XDP" , 267 "rlock-AF_MAX" 268 }; 269 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 270 "wlock-AF_UNSPEC", "wlock-AF_UNIX" , "wlock-AF_INET" , 271 "wlock-AF_AX25" , "wlock-AF_IPX" , "wlock-AF_APPLETALK", 272 "wlock-AF_NETROM", "wlock-AF_BRIDGE" , "wlock-AF_ATMPVC" , 273 "wlock-AF_X25" , "wlock-AF_INET6" , "wlock-AF_ROSE" , 274 "wlock-AF_DECnet", "wlock-AF_NETBEUI" , "wlock-AF_SECURITY" , 275 "wlock-AF_KEY" , "wlock-AF_NETLINK" , "wlock-AF_PACKET" , 276 "wlock-AF_ASH" , "wlock-AF_ECONET" , "wlock-AF_ATMSVC" , 277 "wlock-AF_RDS" , "wlock-AF_SNA" , "wlock-AF_IRDA" , 278 "wlock-AF_PPPOX" , "wlock-AF_WANPIPE" , "wlock-AF_LLC" , 279 "wlock-27" , "wlock-28" , "wlock-AF_CAN" , 280 "wlock-AF_TIPC" , "wlock-AF_BLUETOOTH", "wlock-AF_IUCV" , 281 "wlock-AF_RXRPC" , "wlock-AF_ISDN" , "wlock-AF_PHONET" , 282 "wlock-AF_IEEE802154", "wlock-AF_CAIF" , "wlock-AF_ALG" , 283 "wlock-AF_NFC" , "wlock-AF_VSOCK" , "wlock-AF_KCM" , 284 "wlock-AF_QIPCRTR", "wlock-AF_SMC" , "wlock-AF_XDP" , 285 "wlock-AF_MAX" 286 }; 287 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 288 "elock-AF_UNSPEC", "elock-AF_UNIX" , "elock-AF_INET" , 289 "elock-AF_AX25" , "elock-AF_IPX" , "elock-AF_APPLETALK", 290 "elock-AF_NETROM", "elock-AF_BRIDGE" , "elock-AF_ATMPVC" , 291 "elock-AF_X25" , "elock-AF_INET6" , "elock-AF_ROSE" , 292 "elock-AF_DECnet", "elock-AF_NETBEUI" , "elock-AF_SECURITY" , 293 "elock-AF_KEY" , "elock-AF_NETLINK" , "elock-AF_PACKET" , 294 "elock-AF_ASH" , "elock-AF_ECONET" , "elock-AF_ATMSVC" , 295 "elock-AF_RDS" , "elock-AF_SNA" , "elock-AF_IRDA" , 296 "elock-AF_PPPOX" , "elock-AF_WANPIPE" , "elock-AF_LLC" , 297 "elock-27" , "elock-28" , "elock-AF_CAN" , 298 "elock-AF_TIPC" , "elock-AF_BLUETOOTH", "elock-AF_IUCV" , 299 "elock-AF_RXRPC" , "elock-AF_ISDN" , "elock-AF_PHONET" , 300 "elock-AF_IEEE802154", "elock-AF_CAIF" , "elock-AF_ALG" , 301 "elock-AF_NFC" , "elock-AF_VSOCK" , "elock-AF_KCM" , 302 "elock-AF_QIPCRTR", "elock-AF_SMC" , "elock-AF_XDP" , 303 "elock-AF_MAX" 304 }; 305 306 /* 307 * sk_callback_lock and sk queues locking rules are per-address-family, 308 * so split the lock classes by using a per-AF key: 309 */ 310 static struct lock_class_key af_callback_keys[AF_MAX]; 311 static struct lock_class_key af_rlock_keys[AF_MAX]; 312 static struct lock_class_key af_wlock_keys[AF_MAX]; 313 static struct lock_class_key af_elock_keys[AF_MAX]; 314 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 315 316 /* Run time adjustable parameters. */ 317 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 318 EXPORT_SYMBOL(sysctl_wmem_max); 319 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 320 EXPORT_SYMBOL(sysctl_rmem_max); 321 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 322 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 323 324 /* Maximal space eaten by iovec or ancillary data plus some space */ 325 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 326 EXPORT_SYMBOL(sysctl_optmem_max); 327 328 int sysctl_tstamp_allow_data __read_mostly = 1; 329 330 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 331 EXPORT_SYMBOL_GPL(memalloc_socks_key); 332 333 /** 334 * sk_set_memalloc - sets %SOCK_MEMALLOC 335 * @sk: socket to set it on 336 * 337 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 338 * It's the responsibility of the admin to adjust min_free_kbytes 339 * to meet the requirements 340 */ 341 void sk_set_memalloc(struct sock *sk) 342 { 343 sock_set_flag(sk, SOCK_MEMALLOC); 344 sk->sk_allocation |= __GFP_MEMALLOC; 345 static_branch_inc(&memalloc_socks_key); 346 } 347 EXPORT_SYMBOL_GPL(sk_set_memalloc); 348 349 void sk_clear_memalloc(struct sock *sk) 350 { 351 sock_reset_flag(sk, SOCK_MEMALLOC); 352 sk->sk_allocation &= ~__GFP_MEMALLOC; 353 static_branch_dec(&memalloc_socks_key); 354 355 /* 356 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 357 * progress of swapping. SOCK_MEMALLOC may be cleared while 358 * it has rmem allocations due to the last swapfile being deactivated 359 * but there is a risk that the socket is unusable due to exceeding 360 * the rmem limits. Reclaim the reserves and obey rmem limits again. 361 */ 362 sk_mem_reclaim(sk); 363 } 364 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 365 366 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 367 { 368 int ret; 369 unsigned int noreclaim_flag; 370 371 /* these should have been dropped before queueing */ 372 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 373 374 noreclaim_flag = memalloc_noreclaim_save(); 375 ret = sk->sk_backlog_rcv(sk, skb); 376 memalloc_noreclaim_restore(noreclaim_flag); 377 378 return ret; 379 } 380 EXPORT_SYMBOL(__sk_backlog_rcv); 381 382 static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen) 383 { 384 struct timeval tv; 385 386 if (optlen < sizeof(tv)) 387 return -EINVAL; 388 if (copy_from_user(&tv, optval, sizeof(tv))) 389 return -EFAULT; 390 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 391 return -EDOM; 392 393 if (tv.tv_sec < 0) { 394 static int warned __read_mostly; 395 396 *timeo_p = 0; 397 if (warned < 10 && net_ratelimit()) { 398 warned++; 399 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 400 __func__, current->comm, task_pid_nr(current)); 401 } 402 return 0; 403 } 404 *timeo_p = MAX_SCHEDULE_TIMEOUT; 405 if (tv.tv_sec == 0 && tv.tv_usec == 0) 406 return 0; 407 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1)) 408 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ); 409 return 0; 410 } 411 412 static void sock_warn_obsolete_bsdism(const char *name) 413 { 414 static int warned; 415 static char warncomm[TASK_COMM_LEN]; 416 if (strcmp(warncomm, current->comm) && warned < 5) { 417 strcpy(warncomm, current->comm); 418 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n", 419 warncomm, name); 420 warned++; 421 } 422 } 423 424 static bool sock_needs_netstamp(const struct sock *sk) 425 { 426 switch (sk->sk_family) { 427 case AF_UNSPEC: 428 case AF_UNIX: 429 return false; 430 default: 431 return true; 432 } 433 } 434 435 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 436 { 437 if (sk->sk_flags & flags) { 438 sk->sk_flags &= ~flags; 439 if (sock_needs_netstamp(sk) && 440 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 441 net_disable_timestamp(); 442 } 443 } 444 445 446 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 447 { 448 unsigned long flags; 449 struct sk_buff_head *list = &sk->sk_receive_queue; 450 451 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { 452 atomic_inc(&sk->sk_drops); 453 trace_sock_rcvqueue_full(sk, skb); 454 return -ENOMEM; 455 } 456 457 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 458 atomic_inc(&sk->sk_drops); 459 return -ENOBUFS; 460 } 461 462 skb->dev = NULL; 463 skb_set_owner_r(skb, sk); 464 465 /* we escape from rcu protected region, make sure we dont leak 466 * a norefcounted dst 467 */ 468 skb_dst_force(skb); 469 470 spin_lock_irqsave(&list->lock, flags); 471 sock_skb_set_dropcount(sk, skb); 472 __skb_queue_tail(list, skb); 473 spin_unlock_irqrestore(&list->lock, flags); 474 475 if (!sock_flag(sk, SOCK_DEAD)) 476 sk->sk_data_ready(sk); 477 return 0; 478 } 479 EXPORT_SYMBOL(__sock_queue_rcv_skb); 480 481 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 482 { 483 int err; 484 485 err = sk_filter(sk, skb); 486 if (err) 487 return err; 488 489 return __sock_queue_rcv_skb(sk, skb); 490 } 491 EXPORT_SYMBOL(sock_queue_rcv_skb); 492 493 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 494 const int nested, unsigned int trim_cap, bool refcounted) 495 { 496 int rc = NET_RX_SUCCESS; 497 498 if (sk_filter_trim_cap(sk, skb, trim_cap)) 499 goto discard_and_relse; 500 501 skb->dev = NULL; 502 503 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) { 504 atomic_inc(&sk->sk_drops); 505 goto discard_and_relse; 506 } 507 if (nested) 508 bh_lock_sock_nested(sk); 509 else 510 bh_lock_sock(sk); 511 if (!sock_owned_by_user(sk)) { 512 /* 513 * trylock + unlock semantics: 514 */ 515 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 516 517 rc = sk_backlog_rcv(sk, skb); 518 519 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_); 520 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) { 521 bh_unlock_sock(sk); 522 atomic_inc(&sk->sk_drops); 523 goto discard_and_relse; 524 } 525 526 bh_unlock_sock(sk); 527 out: 528 if (refcounted) 529 sock_put(sk); 530 return rc; 531 discard_and_relse: 532 kfree_skb(skb); 533 goto out; 534 } 535 EXPORT_SYMBOL(__sk_receive_skb); 536 537 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 538 { 539 struct dst_entry *dst = __sk_dst_get(sk); 540 541 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 542 sk_tx_queue_clear(sk); 543 sk->sk_dst_pending_confirm = 0; 544 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 545 dst_release(dst); 546 return NULL; 547 } 548 549 return dst; 550 } 551 EXPORT_SYMBOL(__sk_dst_check); 552 553 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 554 { 555 struct dst_entry *dst = sk_dst_get(sk); 556 557 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 558 sk_dst_reset(sk); 559 dst_release(dst); 560 return NULL; 561 } 562 563 return dst; 564 } 565 EXPORT_SYMBOL(sk_dst_check); 566 567 static int sock_setbindtodevice(struct sock *sk, char __user *optval, 568 int optlen) 569 { 570 int ret = -ENOPROTOOPT; 571 #ifdef CONFIG_NETDEVICES 572 struct net *net = sock_net(sk); 573 char devname[IFNAMSIZ]; 574 int index; 575 576 /* Sorry... */ 577 ret = -EPERM; 578 if (!ns_capable(net->user_ns, CAP_NET_RAW)) 579 goto out; 580 581 ret = -EINVAL; 582 if (optlen < 0) 583 goto out; 584 585 /* Bind this socket to a particular device like "eth0", 586 * as specified in the passed interface name. If the 587 * name is "" or the option length is zero the socket 588 * is not bound. 589 */ 590 if (optlen > IFNAMSIZ - 1) 591 optlen = IFNAMSIZ - 1; 592 memset(devname, 0, sizeof(devname)); 593 594 ret = -EFAULT; 595 if (copy_from_user(devname, optval, optlen)) 596 goto out; 597 598 index = 0; 599 if (devname[0] != '\0') { 600 struct net_device *dev; 601 602 rcu_read_lock(); 603 dev = dev_get_by_name_rcu(net, devname); 604 if (dev) 605 index = dev->ifindex; 606 rcu_read_unlock(); 607 ret = -ENODEV; 608 if (!dev) 609 goto out; 610 } 611 612 lock_sock(sk); 613 sk->sk_bound_dev_if = index; 614 sk_dst_reset(sk); 615 release_sock(sk); 616 617 ret = 0; 618 619 out: 620 #endif 621 622 return ret; 623 } 624 625 static int sock_getbindtodevice(struct sock *sk, char __user *optval, 626 int __user *optlen, int len) 627 { 628 int ret = -ENOPROTOOPT; 629 #ifdef CONFIG_NETDEVICES 630 struct net *net = sock_net(sk); 631 char devname[IFNAMSIZ]; 632 633 if (sk->sk_bound_dev_if == 0) { 634 len = 0; 635 goto zero; 636 } 637 638 ret = -EINVAL; 639 if (len < IFNAMSIZ) 640 goto out; 641 642 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if); 643 if (ret) 644 goto out; 645 646 len = strlen(devname) + 1; 647 648 ret = -EFAULT; 649 if (copy_to_user(optval, devname, len)) 650 goto out; 651 652 zero: 653 ret = -EFAULT; 654 if (put_user(len, optlen)) 655 goto out; 656 657 ret = 0; 658 659 out: 660 #endif 661 662 return ret; 663 } 664 665 static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool) 666 { 667 if (valbool) 668 sock_set_flag(sk, bit); 669 else 670 sock_reset_flag(sk, bit); 671 } 672 673 bool sk_mc_loop(struct sock *sk) 674 { 675 if (dev_recursion_level()) 676 return false; 677 if (!sk) 678 return true; 679 switch (sk->sk_family) { 680 case AF_INET: 681 return inet_sk(sk)->mc_loop; 682 #if IS_ENABLED(CONFIG_IPV6) 683 case AF_INET6: 684 return inet6_sk(sk)->mc_loop; 685 #endif 686 } 687 WARN_ON(1); 688 return true; 689 } 690 EXPORT_SYMBOL(sk_mc_loop); 691 692 /* 693 * This is meant for all protocols to use and covers goings on 694 * at the socket level. Everything here is generic. 695 */ 696 697 int sock_setsockopt(struct socket *sock, int level, int optname, 698 char __user *optval, unsigned int optlen) 699 { 700 struct sock *sk = sock->sk; 701 int val; 702 int valbool; 703 struct linger ling; 704 int ret = 0; 705 706 /* 707 * Options without arguments 708 */ 709 710 if (optname == SO_BINDTODEVICE) 711 return sock_setbindtodevice(sk, optval, optlen); 712 713 if (optlen < sizeof(int)) 714 return -EINVAL; 715 716 if (get_user(val, (int __user *)optval)) 717 return -EFAULT; 718 719 valbool = val ? 1 : 0; 720 721 lock_sock(sk); 722 723 switch (optname) { 724 case SO_DEBUG: 725 if (val && !capable(CAP_NET_ADMIN)) 726 ret = -EACCES; 727 else 728 sock_valbool_flag(sk, SOCK_DBG, valbool); 729 break; 730 case SO_REUSEADDR: 731 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 732 break; 733 case SO_REUSEPORT: 734 sk->sk_reuseport = valbool; 735 break; 736 case SO_TYPE: 737 case SO_PROTOCOL: 738 case SO_DOMAIN: 739 case SO_ERROR: 740 ret = -ENOPROTOOPT; 741 break; 742 case SO_DONTROUTE: 743 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 744 break; 745 case SO_BROADCAST: 746 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 747 break; 748 case SO_SNDBUF: 749 /* Don't error on this BSD doesn't and if you think 750 * about it this is right. Otherwise apps have to 751 * play 'guess the biggest size' games. RCVBUF/SNDBUF 752 * are treated in BSD as hints 753 */ 754 val = min_t(u32, val, sysctl_wmem_max); 755 set_sndbuf: 756 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 757 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); 758 /* Wake up sending tasks if we upped the value. */ 759 sk->sk_write_space(sk); 760 break; 761 762 case SO_SNDBUFFORCE: 763 if (!capable(CAP_NET_ADMIN)) { 764 ret = -EPERM; 765 break; 766 } 767 goto set_sndbuf; 768 769 case SO_RCVBUF: 770 /* Don't error on this BSD doesn't and if you think 771 * about it this is right. Otherwise apps have to 772 * play 'guess the biggest size' games. RCVBUF/SNDBUF 773 * are treated in BSD as hints 774 */ 775 val = min_t(u32, val, sysctl_rmem_max); 776 set_rcvbuf: 777 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 778 /* 779 * We double it on the way in to account for 780 * "struct sk_buff" etc. overhead. Applications 781 * assume that the SO_RCVBUF setting they make will 782 * allow that much actual data to be received on that 783 * socket. 784 * 785 * Applications are unaware that "struct sk_buff" and 786 * other overheads allocate from the receive buffer 787 * during socket buffer allocation. 788 * 789 * And after considering the possible alternatives, 790 * returning the value we actually used in getsockopt 791 * is the most desirable behavior. 792 */ 793 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF); 794 break; 795 796 case SO_RCVBUFFORCE: 797 if (!capable(CAP_NET_ADMIN)) { 798 ret = -EPERM; 799 break; 800 } 801 goto set_rcvbuf; 802 803 case SO_KEEPALIVE: 804 if (sk->sk_prot->keepalive) 805 sk->sk_prot->keepalive(sk, valbool); 806 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 807 break; 808 809 case SO_OOBINLINE: 810 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 811 break; 812 813 case SO_NO_CHECK: 814 sk->sk_no_check_tx = valbool; 815 break; 816 817 case SO_PRIORITY: 818 if ((val >= 0 && val <= 6) || 819 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 820 sk->sk_priority = val; 821 else 822 ret = -EPERM; 823 break; 824 825 case SO_LINGER: 826 if (optlen < sizeof(ling)) { 827 ret = -EINVAL; /* 1003.1g */ 828 break; 829 } 830 if (copy_from_user(&ling, optval, sizeof(ling))) { 831 ret = -EFAULT; 832 break; 833 } 834 if (!ling.l_onoff) 835 sock_reset_flag(sk, SOCK_LINGER); 836 else { 837 #if (BITS_PER_LONG == 32) 838 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ) 839 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT; 840 else 841 #endif 842 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ; 843 sock_set_flag(sk, SOCK_LINGER); 844 } 845 break; 846 847 case SO_BSDCOMPAT: 848 sock_warn_obsolete_bsdism("setsockopt"); 849 break; 850 851 case SO_PASSCRED: 852 if (valbool) 853 set_bit(SOCK_PASSCRED, &sock->flags); 854 else 855 clear_bit(SOCK_PASSCRED, &sock->flags); 856 break; 857 858 case SO_TIMESTAMP: 859 case SO_TIMESTAMPNS: 860 if (valbool) { 861 if (optname == SO_TIMESTAMP) 862 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 863 else 864 sock_set_flag(sk, SOCK_RCVTSTAMPNS); 865 sock_set_flag(sk, SOCK_RCVTSTAMP); 866 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 867 } else { 868 sock_reset_flag(sk, SOCK_RCVTSTAMP); 869 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 870 } 871 break; 872 873 case SO_TIMESTAMPING: 874 if (val & ~SOF_TIMESTAMPING_MASK) { 875 ret = -EINVAL; 876 break; 877 } 878 879 if (val & SOF_TIMESTAMPING_OPT_ID && 880 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 881 if (sk->sk_protocol == IPPROTO_TCP && 882 sk->sk_type == SOCK_STREAM) { 883 if ((1 << sk->sk_state) & 884 (TCPF_CLOSE | TCPF_LISTEN)) { 885 ret = -EINVAL; 886 break; 887 } 888 sk->sk_tskey = tcp_sk(sk)->snd_una; 889 } else { 890 sk->sk_tskey = 0; 891 } 892 } 893 894 if (val & SOF_TIMESTAMPING_OPT_STATS && 895 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) { 896 ret = -EINVAL; 897 break; 898 } 899 900 sk->sk_tsflags = val; 901 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 902 sock_enable_timestamp(sk, 903 SOCK_TIMESTAMPING_RX_SOFTWARE); 904 else 905 sock_disable_timestamp(sk, 906 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 907 break; 908 909 case SO_RCVLOWAT: 910 if (val < 0) 911 val = INT_MAX; 912 if (sock->ops->set_rcvlowat) 913 ret = sock->ops->set_rcvlowat(sk, val); 914 else 915 sk->sk_rcvlowat = val ? : 1; 916 break; 917 918 case SO_RCVTIMEO: 919 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen); 920 break; 921 922 case SO_SNDTIMEO: 923 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen); 924 break; 925 926 case SO_ATTACH_FILTER: 927 ret = -EINVAL; 928 if (optlen == sizeof(struct sock_fprog)) { 929 struct sock_fprog fprog; 930 931 ret = -EFAULT; 932 if (copy_from_user(&fprog, optval, sizeof(fprog))) 933 break; 934 935 ret = sk_attach_filter(&fprog, sk); 936 } 937 break; 938 939 case SO_ATTACH_BPF: 940 ret = -EINVAL; 941 if (optlen == sizeof(u32)) { 942 u32 ufd; 943 944 ret = -EFAULT; 945 if (copy_from_user(&ufd, optval, sizeof(ufd))) 946 break; 947 948 ret = sk_attach_bpf(ufd, sk); 949 } 950 break; 951 952 case SO_ATTACH_REUSEPORT_CBPF: 953 ret = -EINVAL; 954 if (optlen == sizeof(struct sock_fprog)) { 955 struct sock_fprog fprog; 956 957 ret = -EFAULT; 958 if (copy_from_user(&fprog, optval, sizeof(fprog))) 959 break; 960 961 ret = sk_reuseport_attach_filter(&fprog, sk); 962 } 963 break; 964 965 case SO_ATTACH_REUSEPORT_EBPF: 966 ret = -EINVAL; 967 if (optlen == sizeof(u32)) { 968 u32 ufd; 969 970 ret = -EFAULT; 971 if (copy_from_user(&ufd, optval, sizeof(ufd))) 972 break; 973 974 ret = sk_reuseport_attach_bpf(ufd, sk); 975 } 976 break; 977 978 case SO_DETACH_FILTER: 979 ret = sk_detach_filter(sk); 980 break; 981 982 case SO_LOCK_FILTER: 983 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 984 ret = -EPERM; 985 else 986 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 987 break; 988 989 case SO_PASSSEC: 990 if (valbool) 991 set_bit(SOCK_PASSSEC, &sock->flags); 992 else 993 clear_bit(SOCK_PASSSEC, &sock->flags); 994 break; 995 case SO_MARK: 996 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 997 ret = -EPERM; 998 else 999 sk->sk_mark = val; 1000 break; 1001 1002 case SO_RXQ_OVFL: 1003 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1004 break; 1005 1006 case SO_WIFI_STATUS: 1007 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1008 break; 1009 1010 case SO_PEEK_OFF: 1011 if (sock->ops->set_peek_off) 1012 ret = sock->ops->set_peek_off(sk, val); 1013 else 1014 ret = -EOPNOTSUPP; 1015 break; 1016 1017 case SO_NOFCS: 1018 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1019 break; 1020 1021 case SO_SELECT_ERR_QUEUE: 1022 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1023 break; 1024 1025 #ifdef CONFIG_NET_RX_BUSY_POLL 1026 case SO_BUSY_POLL: 1027 /* allow unprivileged users to decrease the value */ 1028 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN)) 1029 ret = -EPERM; 1030 else { 1031 if (val < 0) 1032 ret = -EINVAL; 1033 else 1034 sk->sk_ll_usec = val; 1035 } 1036 break; 1037 #endif 1038 1039 case SO_MAX_PACING_RATE: 1040 if (val != ~0U) 1041 cmpxchg(&sk->sk_pacing_status, 1042 SK_PACING_NONE, 1043 SK_PACING_NEEDED); 1044 sk->sk_max_pacing_rate = val; 1045 sk->sk_pacing_rate = min(sk->sk_pacing_rate, 1046 sk->sk_max_pacing_rate); 1047 break; 1048 1049 case SO_INCOMING_CPU: 1050 sk->sk_incoming_cpu = val; 1051 break; 1052 1053 case SO_CNX_ADVICE: 1054 if (val == 1) 1055 dst_negative_advice(sk); 1056 break; 1057 1058 case SO_ZEROCOPY: 1059 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1060 if (sk->sk_protocol != IPPROTO_TCP) 1061 ret = -ENOTSUPP; 1062 } else if (sk->sk_family != PF_RDS) { 1063 ret = -ENOTSUPP; 1064 } 1065 if (!ret) { 1066 if (val < 0 || val > 1) 1067 ret = -EINVAL; 1068 else 1069 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1070 } 1071 break; 1072 1073 default: 1074 ret = -ENOPROTOOPT; 1075 break; 1076 } 1077 release_sock(sk); 1078 return ret; 1079 } 1080 EXPORT_SYMBOL(sock_setsockopt); 1081 1082 1083 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1084 struct ucred *ucred) 1085 { 1086 ucred->pid = pid_vnr(pid); 1087 ucred->uid = ucred->gid = -1; 1088 if (cred) { 1089 struct user_namespace *current_ns = current_user_ns(); 1090 1091 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1092 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1093 } 1094 } 1095 1096 static int groups_to_user(gid_t __user *dst, const struct group_info *src) 1097 { 1098 struct user_namespace *user_ns = current_user_ns(); 1099 int i; 1100 1101 for (i = 0; i < src->ngroups; i++) 1102 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i)) 1103 return -EFAULT; 1104 1105 return 0; 1106 } 1107 1108 int sock_getsockopt(struct socket *sock, int level, int optname, 1109 char __user *optval, int __user *optlen) 1110 { 1111 struct sock *sk = sock->sk; 1112 1113 union { 1114 int val; 1115 u64 val64; 1116 struct linger ling; 1117 struct timeval tm; 1118 } v; 1119 1120 int lv = sizeof(int); 1121 int len; 1122 1123 if (get_user(len, optlen)) 1124 return -EFAULT; 1125 if (len < 0) 1126 return -EINVAL; 1127 1128 memset(&v, 0, sizeof(v)); 1129 1130 switch (optname) { 1131 case SO_DEBUG: 1132 v.val = sock_flag(sk, SOCK_DBG); 1133 break; 1134 1135 case SO_DONTROUTE: 1136 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1137 break; 1138 1139 case SO_BROADCAST: 1140 v.val = sock_flag(sk, SOCK_BROADCAST); 1141 break; 1142 1143 case SO_SNDBUF: 1144 v.val = sk->sk_sndbuf; 1145 break; 1146 1147 case SO_RCVBUF: 1148 v.val = sk->sk_rcvbuf; 1149 break; 1150 1151 case SO_REUSEADDR: 1152 v.val = sk->sk_reuse; 1153 break; 1154 1155 case SO_REUSEPORT: 1156 v.val = sk->sk_reuseport; 1157 break; 1158 1159 case SO_KEEPALIVE: 1160 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1161 break; 1162 1163 case SO_TYPE: 1164 v.val = sk->sk_type; 1165 break; 1166 1167 case SO_PROTOCOL: 1168 v.val = sk->sk_protocol; 1169 break; 1170 1171 case SO_DOMAIN: 1172 v.val = sk->sk_family; 1173 break; 1174 1175 case SO_ERROR: 1176 v.val = -sock_error(sk); 1177 if (v.val == 0) 1178 v.val = xchg(&sk->sk_err_soft, 0); 1179 break; 1180 1181 case SO_OOBINLINE: 1182 v.val = sock_flag(sk, SOCK_URGINLINE); 1183 break; 1184 1185 case SO_NO_CHECK: 1186 v.val = sk->sk_no_check_tx; 1187 break; 1188 1189 case SO_PRIORITY: 1190 v.val = sk->sk_priority; 1191 break; 1192 1193 case SO_LINGER: 1194 lv = sizeof(v.ling); 1195 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1196 v.ling.l_linger = sk->sk_lingertime / HZ; 1197 break; 1198 1199 case SO_BSDCOMPAT: 1200 sock_warn_obsolete_bsdism("getsockopt"); 1201 break; 1202 1203 case SO_TIMESTAMP: 1204 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1205 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1206 break; 1207 1208 case SO_TIMESTAMPNS: 1209 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS); 1210 break; 1211 1212 case SO_TIMESTAMPING: 1213 v.val = sk->sk_tsflags; 1214 break; 1215 1216 case SO_RCVTIMEO: 1217 lv = sizeof(struct timeval); 1218 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) { 1219 v.tm.tv_sec = 0; 1220 v.tm.tv_usec = 0; 1221 } else { 1222 v.tm.tv_sec = sk->sk_rcvtimeo / HZ; 1223 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ; 1224 } 1225 break; 1226 1227 case SO_SNDTIMEO: 1228 lv = sizeof(struct timeval); 1229 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) { 1230 v.tm.tv_sec = 0; 1231 v.tm.tv_usec = 0; 1232 } else { 1233 v.tm.tv_sec = sk->sk_sndtimeo / HZ; 1234 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ; 1235 } 1236 break; 1237 1238 case SO_RCVLOWAT: 1239 v.val = sk->sk_rcvlowat; 1240 break; 1241 1242 case SO_SNDLOWAT: 1243 v.val = 1; 1244 break; 1245 1246 case SO_PASSCRED: 1247 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1248 break; 1249 1250 case SO_PEERCRED: 1251 { 1252 struct ucred peercred; 1253 if (len > sizeof(peercred)) 1254 len = sizeof(peercred); 1255 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1256 if (copy_to_user(optval, &peercred, len)) 1257 return -EFAULT; 1258 goto lenout; 1259 } 1260 1261 case SO_PEERGROUPS: 1262 { 1263 int ret, n; 1264 1265 if (!sk->sk_peer_cred) 1266 return -ENODATA; 1267 1268 n = sk->sk_peer_cred->group_info->ngroups; 1269 if (len < n * sizeof(gid_t)) { 1270 len = n * sizeof(gid_t); 1271 return put_user(len, optlen) ? -EFAULT : -ERANGE; 1272 } 1273 len = n * sizeof(gid_t); 1274 1275 ret = groups_to_user((gid_t __user *)optval, 1276 sk->sk_peer_cred->group_info); 1277 if (ret) 1278 return ret; 1279 goto lenout; 1280 } 1281 1282 case SO_PEERNAME: 1283 { 1284 char address[128]; 1285 1286 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2); 1287 if (lv < 0) 1288 return -ENOTCONN; 1289 if (lv < len) 1290 return -EINVAL; 1291 if (copy_to_user(optval, address, len)) 1292 return -EFAULT; 1293 goto lenout; 1294 } 1295 1296 /* Dubious BSD thing... Probably nobody even uses it, but 1297 * the UNIX standard wants it for whatever reason... -DaveM 1298 */ 1299 case SO_ACCEPTCONN: 1300 v.val = sk->sk_state == TCP_LISTEN; 1301 break; 1302 1303 case SO_PASSSEC: 1304 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1305 break; 1306 1307 case SO_PEERSEC: 1308 return security_socket_getpeersec_stream(sock, optval, optlen, len); 1309 1310 case SO_MARK: 1311 v.val = sk->sk_mark; 1312 break; 1313 1314 case SO_RXQ_OVFL: 1315 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1316 break; 1317 1318 case SO_WIFI_STATUS: 1319 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1320 break; 1321 1322 case SO_PEEK_OFF: 1323 if (!sock->ops->set_peek_off) 1324 return -EOPNOTSUPP; 1325 1326 v.val = sk->sk_peek_off; 1327 break; 1328 case SO_NOFCS: 1329 v.val = sock_flag(sk, SOCK_NOFCS); 1330 break; 1331 1332 case SO_BINDTODEVICE: 1333 return sock_getbindtodevice(sk, optval, optlen, len); 1334 1335 case SO_GET_FILTER: 1336 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len); 1337 if (len < 0) 1338 return len; 1339 1340 goto lenout; 1341 1342 case SO_LOCK_FILTER: 1343 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1344 break; 1345 1346 case SO_BPF_EXTENSIONS: 1347 v.val = bpf_tell_extensions(); 1348 break; 1349 1350 case SO_SELECT_ERR_QUEUE: 1351 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1352 break; 1353 1354 #ifdef CONFIG_NET_RX_BUSY_POLL 1355 case SO_BUSY_POLL: 1356 v.val = sk->sk_ll_usec; 1357 break; 1358 #endif 1359 1360 case SO_MAX_PACING_RATE: 1361 v.val = sk->sk_max_pacing_rate; 1362 break; 1363 1364 case SO_INCOMING_CPU: 1365 v.val = sk->sk_incoming_cpu; 1366 break; 1367 1368 case SO_MEMINFO: 1369 { 1370 u32 meminfo[SK_MEMINFO_VARS]; 1371 1372 if (get_user(len, optlen)) 1373 return -EFAULT; 1374 1375 sk_get_meminfo(sk, meminfo); 1376 1377 len = min_t(unsigned int, len, sizeof(meminfo)); 1378 if (copy_to_user(optval, &meminfo, len)) 1379 return -EFAULT; 1380 1381 goto lenout; 1382 } 1383 1384 #ifdef CONFIG_NET_RX_BUSY_POLL 1385 case SO_INCOMING_NAPI_ID: 1386 v.val = READ_ONCE(sk->sk_napi_id); 1387 1388 /* aggregate non-NAPI IDs down to 0 */ 1389 if (v.val < MIN_NAPI_ID) 1390 v.val = 0; 1391 1392 break; 1393 #endif 1394 1395 case SO_COOKIE: 1396 lv = sizeof(u64); 1397 if (len < lv) 1398 return -EINVAL; 1399 v.val64 = sock_gen_cookie(sk); 1400 break; 1401 1402 case SO_ZEROCOPY: 1403 v.val = sock_flag(sk, SOCK_ZEROCOPY); 1404 break; 1405 1406 default: 1407 /* We implement the SO_SNDLOWAT etc to not be settable 1408 * (1003.1g 7). 1409 */ 1410 return -ENOPROTOOPT; 1411 } 1412 1413 if (len > lv) 1414 len = lv; 1415 if (copy_to_user(optval, &v, len)) 1416 return -EFAULT; 1417 lenout: 1418 if (put_user(len, optlen)) 1419 return -EFAULT; 1420 return 0; 1421 } 1422 1423 /* 1424 * Initialize an sk_lock. 1425 * 1426 * (We also register the sk_lock with the lock validator.) 1427 */ 1428 static inline void sock_lock_init(struct sock *sk) 1429 { 1430 if (sk->sk_kern_sock) 1431 sock_lock_init_class_and_name( 1432 sk, 1433 af_family_kern_slock_key_strings[sk->sk_family], 1434 af_family_kern_slock_keys + sk->sk_family, 1435 af_family_kern_key_strings[sk->sk_family], 1436 af_family_kern_keys + sk->sk_family); 1437 else 1438 sock_lock_init_class_and_name( 1439 sk, 1440 af_family_slock_key_strings[sk->sk_family], 1441 af_family_slock_keys + sk->sk_family, 1442 af_family_key_strings[sk->sk_family], 1443 af_family_keys + sk->sk_family); 1444 } 1445 1446 /* 1447 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 1448 * even temporarly, because of RCU lookups. sk_node should also be left as is. 1449 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 1450 */ 1451 static void sock_copy(struct sock *nsk, const struct sock *osk) 1452 { 1453 #ifdef CONFIG_SECURITY_NETWORK 1454 void *sptr = nsk->sk_security; 1455 #endif 1456 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 1457 1458 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 1459 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 1460 1461 #ifdef CONFIG_SECURITY_NETWORK 1462 nsk->sk_security = sptr; 1463 security_sk_clone(osk, nsk); 1464 #endif 1465 } 1466 1467 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 1468 int family) 1469 { 1470 struct sock *sk; 1471 struct kmem_cache *slab; 1472 1473 slab = prot->slab; 1474 if (slab != NULL) { 1475 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 1476 if (!sk) 1477 return sk; 1478 if (priority & __GFP_ZERO) 1479 sk_prot_clear_nulls(sk, prot->obj_size); 1480 } else 1481 sk = kmalloc(prot->obj_size, priority); 1482 1483 if (sk != NULL) { 1484 if (security_sk_alloc(sk, family, priority)) 1485 goto out_free; 1486 1487 if (!try_module_get(prot->owner)) 1488 goto out_free_sec; 1489 sk_tx_queue_clear(sk); 1490 } 1491 1492 return sk; 1493 1494 out_free_sec: 1495 security_sk_free(sk); 1496 out_free: 1497 if (slab != NULL) 1498 kmem_cache_free(slab, sk); 1499 else 1500 kfree(sk); 1501 return NULL; 1502 } 1503 1504 static void sk_prot_free(struct proto *prot, struct sock *sk) 1505 { 1506 struct kmem_cache *slab; 1507 struct module *owner; 1508 1509 owner = prot->owner; 1510 slab = prot->slab; 1511 1512 cgroup_sk_free(&sk->sk_cgrp_data); 1513 mem_cgroup_sk_free(sk); 1514 security_sk_free(sk); 1515 if (slab != NULL) 1516 kmem_cache_free(slab, sk); 1517 else 1518 kfree(sk); 1519 module_put(owner); 1520 } 1521 1522 /** 1523 * sk_alloc - All socket objects are allocated here 1524 * @net: the applicable net namespace 1525 * @family: protocol family 1526 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1527 * @prot: struct proto associated with this new sock instance 1528 * @kern: is this to be a kernel socket? 1529 */ 1530 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 1531 struct proto *prot, int kern) 1532 { 1533 struct sock *sk; 1534 1535 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 1536 if (sk) { 1537 sk->sk_family = family; 1538 /* 1539 * See comment in struct sock definition to understand 1540 * why we need sk_prot_creator -acme 1541 */ 1542 sk->sk_prot = sk->sk_prot_creator = prot; 1543 sk->sk_kern_sock = kern; 1544 sock_lock_init(sk); 1545 sk->sk_net_refcnt = kern ? 0 : 1; 1546 if (likely(sk->sk_net_refcnt)) { 1547 get_net(net); 1548 sock_inuse_add(net, 1); 1549 } 1550 1551 sock_net_set(sk, net); 1552 refcount_set(&sk->sk_wmem_alloc, 1); 1553 1554 mem_cgroup_sk_alloc(sk); 1555 cgroup_sk_alloc(&sk->sk_cgrp_data); 1556 sock_update_classid(&sk->sk_cgrp_data); 1557 sock_update_netprioidx(&sk->sk_cgrp_data); 1558 } 1559 1560 return sk; 1561 } 1562 EXPORT_SYMBOL(sk_alloc); 1563 1564 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 1565 * grace period. This is the case for UDP sockets and TCP listeners. 1566 */ 1567 static void __sk_destruct(struct rcu_head *head) 1568 { 1569 struct sock *sk = container_of(head, struct sock, sk_rcu); 1570 struct sk_filter *filter; 1571 1572 if (sk->sk_destruct) 1573 sk->sk_destruct(sk); 1574 1575 filter = rcu_dereference_check(sk->sk_filter, 1576 refcount_read(&sk->sk_wmem_alloc) == 0); 1577 if (filter) { 1578 sk_filter_uncharge(sk, filter); 1579 RCU_INIT_POINTER(sk->sk_filter, NULL); 1580 } 1581 if (rcu_access_pointer(sk->sk_reuseport_cb)) 1582 reuseport_detach_sock(sk); 1583 1584 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 1585 1586 if (atomic_read(&sk->sk_omem_alloc)) 1587 pr_debug("%s: optmem leakage (%d bytes) detected\n", 1588 __func__, atomic_read(&sk->sk_omem_alloc)); 1589 1590 if (sk->sk_frag.page) { 1591 put_page(sk->sk_frag.page); 1592 sk->sk_frag.page = NULL; 1593 } 1594 1595 if (sk->sk_peer_cred) 1596 put_cred(sk->sk_peer_cred); 1597 put_pid(sk->sk_peer_pid); 1598 if (likely(sk->sk_net_refcnt)) 1599 put_net(sock_net(sk)); 1600 sk_prot_free(sk->sk_prot_creator, sk); 1601 } 1602 1603 void sk_destruct(struct sock *sk) 1604 { 1605 if (sock_flag(sk, SOCK_RCU_FREE)) 1606 call_rcu(&sk->sk_rcu, __sk_destruct); 1607 else 1608 __sk_destruct(&sk->sk_rcu); 1609 } 1610 1611 static void __sk_free(struct sock *sk) 1612 { 1613 if (likely(sk->sk_net_refcnt)) 1614 sock_inuse_add(sock_net(sk), -1); 1615 1616 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 1617 sock_diag_broadcast_destroy(sk); 1618 else 1619 sk_destruct(sk); 1620 } 1621 1622 void sk_free(struct sock *sk) 1623 { 1624 /* 1625 * We subtract one from sk_wmem_alloc and can know if 1626 * some packets are still in some tx queue. 1627 * If not null, sock_wfree() will call __sk_free(sk) later 1628 */ 1629 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 1630 __sk_free(sk); 1631 } 1632 EXPORT_SYMBOL(sk_free); 1633 1634 static void sk_init_common(struct sock *sk) 1635 { 1636 skb_queue_head_init(&sk->sk_receive_queue); 1637 skb_queue_head_init(&sk->sk_write_queue); 1638 skb_queue_head_init(&sk->sk_error_queue); 1639 1640 rwlock_init(&sk->sk_callback_lock); 1641 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 1642 af_rlock_keys + sk->sk_family, 1643 af_family_rlock_key_strings[sk->sk_family]); 1644 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 1645 af_wlock_keys + sk->sk_family, 1646 af_family_wlock_key_strings[sk->sk_family]); 1647 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 1648 af_elock_keys + sk->sk_family, 1649 af_family_elock_key_strings[sk->sk_family]); 1650 lockdep_set_class_and_name(&sk->sk_callback_lock, 1651 af_callback_keys + sk->sk_family, 1652 af_family_clock_key_strings[sk->sk_family]); 1653 } 1654 1655 /** 1656 * sk_clone_lock - clone a socket, and lock its clone 1657 * @sk: the socket to clone 1658 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1659 * 1660 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1661 */ 1662 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 1663 { 1664 struct sock *newsk; 1665 bool is_charged = true; 1666 1667 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family); 1668 if (newsk != NULL) { 1669 struct sk_filter *filter; 1670 1671 sock_copy(newsk, sk); 1672 1673 newsk->sk_prot_creator = sk->sk_prot; 1674 1675 /* SANITY */ 1676 if (likely(newsk->sk_net_refcnt)) 1677 get_net(sock_net(newsk)); 1678 sk_node_init(&newsk->sk_node); 1679 sock_lock_init(newsk); 1680 bh_lock_sock(newsk); 1681 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 1682 newsk->sk_backlog.len = 0; 1683 1684 atomic_set(&newsk->sk_rmem_alloc, 0); 1685 /* 1686 * sk_wmem_alloc set to one (see sk_free() and sock_wfree()) 1687 */ 1688 refcount_set(&newsk->sk_wmem_alloc, 1); 1689 atomic_set(&newsk->sk_omem_alloc, 0); 1690 sk_init_common(newsk); 1691 1692 newsk->sk_dst_cache = NULL; 1693 newsk->sk_dst_pending_confirm = 0; 1694 newsk->sk_wmem_queued = 0; 1695 newsk->sk_forward_alloc = 0; 1696 atomic_set(&newsk->sk_drops, 0); 1697 newsk->sk_send_head = NULL; 1698 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 1699 atomic_set(&newsk->sk_zckey, 0); 1700 1701 sock_reset_flag(newsk, SOCK_DONE); 1702 mem_cgroup_sk_alloc(newsk); 1703 cgroup_sk_alloc(&newsk->sk_cgrp_data); 1704 1705 rcu_read_lock(); 1706 filter = rcu_dereference(sk->sk_filter); 1707 if (filter != NULL) 1708 /* though it's an empty new sock, the charging may fail 1709 * if sysctl_optmem_max was changed between creation of 1710 * original socket and cloning 1711 */ 1712 is_charged = sk_filter_charge(newsk, filter); 1713 RCU_INIT_POINTER(newsk->sk_filter, filter); 1714 rcu_read_unlock(); 1715 1716 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 1717 /* We need to make sure that we don't uncharge the new 1718 * socket if we couldn't charge it in the first place 1719 * as otherwise we uncharge the parent's filter. 1720 */ 1721 if (!is_charged) 1722 RCU_INIT_POINTER(newsk->sk_filter, NULL); 1723 sk_free_unlock_clone(newsk); 1724 newsk = NULL; 1725 goto out; 1726 } 1727 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 1728 1729 newsk->sk_err = 0; 1730 newsk->sk_err_soft = 0; 1731 newsk->sk_priority = 0; 1732 newsk->sk_incoming_cpu = raw_smp_processor_id(); 1733 atomic64_set(&newsk->sk_cookie, 0); 1734 if (likely(newsk->sk_net_refcnt)) 1735 sock_inuse_add(sock_net(newsk), 1); 1736 1737 /* 1738 * Before updating sk_refcnt, we must commit prior changes to memory 1739 * (Documentation/RCU/rculist_nulls.txt for details) 1740 */ 1741 smp_wmb(); 1742 refcount_set(&newsk->sk_refcnt, 2); 1743 1744 /* 1745 * Increment the counter in the same struct proto as the master 1746 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that 1747 * is the same as sk->sk_prot->socks, as this field was copied 1748 * with memcpy). 1749 * 1750 * This _changes_ the previous behaviour, where 1751 * tcp_create_openreq_child always was incrementing the 1752 * equivalent to tcp_prot->socks (inet_sock_nr), so this have 1753 * to be taken into account in all callers. -acme 1754 */ 1755 sk_refcnt_debug_inc(newsk); 1756 sk_set_socket(newsk, NULL); 1757 newsk->sk_wq = NULL; 1758 1759 if (newsk->sk_prot->sockets_allocated) 1760 sk_sockets_allocated_inc(newsk); 1761 1762 if (sock_needs_netstamp(sk) && 1763 newsk->sk_flags & SK_FLAGS_TIMESTAMP) 1764 net_enable_timestamp(); 1765 } 1766 out: 1767 return newsk; 1768 } 1769 EXPORT_SYMBOL_GPL(sk_clone_lock); 1770 1771 void sk_free_unlock_clone(struct sock *sk) 1772 { 1773 /* It is still raw copy of parent, so invalidate 1774 * destructor and make plain sk_free() */ 1775 sk->sk_destruct = NULL; 1776 bh_unlock_sock(sk); 1777 sk_free(sk); 1778 } 1779 EXPORT_SYMBOL_GPL(sk_free_unlock_clone); 1780 1781 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 1782 { 1783 u32 max_segs = 1; 1784 1785 sk_dst_set(sk, dst); 1786 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps; 1787 if (sk->sk_route_caps & NETIF_F_GSO) 1788 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 1789 sk->sk_route_caps &= ~sk->sk_route_nocaps; 1790 if (sk_can_gso(sk)) { 1791 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 1792 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 1793 } else { 1794 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 1795 sk->sk_gso_max_size = dst->dev->gso_max_size; 1796 max_segs = max_t(u32, dst->dev->gso_max_segs, 1); 1797 } 1798 } 1799 sk->sk_gso_max_segs = max_segs; 1800 } 1801 EXPORT_SYMBOL_GPL(sk_setup_caps); 1802 1803 /* 1804 * Simple resource managers for sockets. 1805 */ 1806 1807 1808 /* 1809 * Write buffer destructor automatically called from kfree_skb. 1810 */ 1811 void sock_wfree(struct sk_buff *skb) 1812 { 1813 struct sock *sk = skb->sk; 1814 unsigned int len = skb->truesize; 1815 1816 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 1817 /* 1818 * Keep a reference on sk_wmem_alloc, this will be released 1819 * after sk_write_space() call 1820 */ 1821 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 1822 sk->sk_write_space(sk); 1823 len = 1; 1824 } 1825 /* 1826 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 1827 * could not do because of in-flight packets 1828 */ 1829 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 1830 __sk_free(sk); 1831 } 1832 EXPORT_SYMBOL(sock_wfree); 1833 1834 /* This variant of sock_wfree() is used by TCP, 1835 * since it sets SOCK_USE_WRITE_QUEUE. 1836 */ 1837 void __sock_wfree(struct sk_buff *skb) 1838 { 1839 struct sock *sk = skb->sk; 1840 1841 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 1842 __sk_free(sk); 1843 } 1844 1845 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 1846 { 1847 skb_orphan(skb); 1848 skb->sk = sk; 1849 #ifdef CONFIG_INET 1850 if (unlikely(!sk_fullsock(sk))) { 1851 skb->destructor = sock_edemux; 1852 sock_hold(sk); 1853 return; 1854 } 1855 #endif 1856 skb->destructor = sock_wfree; 1857 skb_set_hash_from_sk(skb, sk); 1858 /* 1859 * We used to take a refcount on sk, but following operation 1860 * is enough to guarantee sk_free() wont free this sock until 1861 * all in-flight packets are completed 1862 */ 1863 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 1864 } 1865 EXPORT_SYMBOL(skb_set_owner_w); 1866 1867 /* This helper is used by netem, as it can hold packets in its 1868 * delay queue. We want to allow the owner socket to send more 1869 * packets, as if they were already TX completed by a typical driver. 1870 * But we also want to keep skb->sk set because some packet schedulers 1871 * rely on it (sch_fq for example). 1872 */ 1873 void skb_orphan_partial(struct sk_buff *skb) 1874 { 1875 if (skb_is_tcp_pure_ack(skb)) 1876 return; 1877 1878 if (skb->destructor == sock_wfree 1879 #ifdef CONFIG_INET 1880 || skb->destructor == tcp_wfree 1881 #endif 1882 ) { 1883 struct sock *sk = skb->sk; 1884 1885 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 1886 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)); 1887 skb->destructor = sock_efree; 1888 } 1889 } else { 1890 skb_orphan(skb); 1891 } 1892 } 1893 EXPORT_SYMBOL(skb_orphan_partial); 1894 1895 /* 1896 * Read buffer destructor automatically called from kfree_skb. 1897 */ 1898 void sock_rfree(struct sk_buff *skb) 1899 { 1900 struct sock *sk = skb->sk; 1901 unsigned int len = skb->truesize; 1902 1903 atomic_sub(len, &sk->sk_rmem_alloc); 1904 sk_mem_uncharge(sk, len); 1905 } 1906 EXPORT_SYMBOL(sock_rfree); 1907 1908 /* 1909 * Buffer destructor for skbs that are not used directly in read or write 1910 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 1911 */ 1912 void sock_efree(struct sk_buff *skb) 1913 { 1914 sock_put(skb->sk); 1915 } 1916 EXPORT_SYMBOL(sock_efree); 1917 1918 kuid_t sock_i_uid(struct sock *sk) 1919 { 1920 kuid_t uid; 1921 1922 read_lock_bh(&sk->sk_callback_lock); 1923 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 1924 read_unlock_bh(&sk->sk_callback_lock); 1925 return uid; 1926 } 1927 EXPORT_SYMBOL(sock_i_uid); 1928 1929 unsigned long sock_i_ino(struct sock *sk) 1930 { 1931 unsigned long ino; 1932 1933 read_lock_bh(&sk->sk_callback_lock); 1934 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 1935 read_unlock_bh(&sk->sk_callback_lock); 1936 return ino; 1937 } 1938 EXPORT_SYMBOL(sock_i_ino); 1939 1940 /* 1941 * Allocate a skb from the socket's send buffer. 1942 */ 1943 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 1944 gfp_t priority) 1945 { 1946 if (force || refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) { 1947 struct sk_buff *skb = alloc_skb(size, priority); 1948 if (skb) { 1949 skb_set_owner_w(skb, sk); 1950 return skb; 1951 } 1952 } 1953 return NULL; 1954 } 1955 EXPORT_SYMBOL(sock_wmalloc); 1956 1957 static void sock_ofree(struct sk_buff *skb) 1958 { 1959 struct sock *sk = skb->sk; 1960 1961 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 1962 } 1963 1964 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 1965 gfp_t priority) 1966 { 1967 struct sk_buff *skb; 1968 1969 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 1970 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 1971 sysctl_optmem_max) 1972 return NULL; 1973 1974 skb = alloc_skb(size, priority); 1975 if (!skb) 1976 return NULL; 1977 1978 atomic_add(skb->truesize, &sk->sk_omem_alloc); 1979 skb->sk = sk; 1980 skb->destructor = sock_ofree; 1981 return skb; 1982 } 1983 1984 /* 1985 * Allocate a memory block from the socket's option memory buffer. 1986 */ 1987 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 1988 { 1989 if ((unsigned int)size <= sysctl_optmem_max && 1990 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { 1991 void *mem; 1992 /* First do the add, to avoid the race if kmalloc 1993 * might sleep. 1994 */ 1995 atomic_add(size, &sk->sk_omem_alloc); 1996 mem = kmalloc(size, priority); 1997 if (mem) 1998 return mem; 1999 atomic_sub(size, &sk->sk_omem_alloc); 2000 } 2001 return NULL; 2002 } 2003 EXPORT_SYMBOL(sock_kmalloc); 2004 2005 /* Free an option memory block. Note, we actually want the inline 2006 * here as this allows gcc to detect the nullify and fold away the 2007 * condition entirely. 2008 */ 2009 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2010 const bool nullify) 2011 { 2012 if (WARN_ON_ONCE(!mem)) 2013 return; 2014 if (nullify) 2015 kzfree(mem); 2016 else 2017 kfree(mem); 2018 atomic_sub(size, &sk->sk_omem_alloc); 2019 } 2020 2021 void sock_kfree_s(struct sock *sk, void *mem, int size) 2022 { 2023 __sock_kfree_s(sk, mem, size, false); 2024 } 2025 EXPORT_SYMBOL(sock_kfree_s); 2026 2027 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2028 { 2029 __sock_kfree_s(sk, mem, size, true); 2030 } 2031 EXPORT_SYMBOL(sock_kzfree_s); 2032 2033 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2034 I think, these locks should be removed for datagram sockets. 2035 */ 2036 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2037 { 2038 DEFINE_WAIT(wait); 2039 2040 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2041 for (;;) { 2042 if (!timeo) 2043 break; 2044 if (signal_pending(current)) 2045 break; 2046 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2047 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2048 if (refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) 2049 break; 2050 if (sk->sk_shutdown & SEND_SHUTDOWN) 2051 break; 2052 if (sk->sk_err) 2053 break; 2054 timeo = schedule_timeout(timeo); 2055 } 2056 finish_wait(sk_sleep(sk), &wait); 2057 return timeo; 2058 } 2059 2060 2061 /* 2062 * Generic send/receive buffer handlers 2063 */ 2064 2065 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2066 unsigned long data_len, int noblock, 2067 int *errcode, int max_page_order) 2068 { 2069 struct sk_buff *skb; 2070 long timeo; 2071 int err; 2072 2073 timeo = sock_sndtimeo(sk, noblock); 2074 for (;;) { 2075 err = sock_error(sk); 2076 if (err != 0) 2077 goto failure; 2078 2079 err = -EPIPE; 2080 if (sk->sk_shutdown & SEND_SHUTDOWN) 2081 goto failure; 2082 2083 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf) 2084 break; 2085 2086 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2087 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2088 err = -EAGAIN; 2089 if (!timeo) 2090 goto failure; 2091 if (signal_pending(current)) 2092 goto interrupted; 2093 timeo = sock_wait_for_wmem(sk, timeo); 2094 } 2095 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 2096 errcode, sk->sk_allocation); 2097 if (skb) 2098 skb_set_owner_w(skb, sk); 2099 return skb; 2100 2101 interrupted: 2102 err = sock_intr_errno(timeo); 2103 failure: 2104 *errcode = err; 2105 return NULL; 2106 } 2107 EXPORT_SYMBOL(sock_alloc_send_pskb); 2108 2109 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size, 2110 int noblock, int *errcode) 2111 { 2112 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0); 2113 } 2114 EXPORT_SYMBOL(sock_alloc_send_skb); 2115 2116 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, 2117 struct sockcm_cookie *sockc) 2118 { 2119 u32 tsflags; 2120 2121 switch (cmsg->cmsg_type) { 2122 case SO_MARK: 2123 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 2124 return -EPERM; 2125 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2126 return -EINVAL; 2127 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 2128 break; 2129 case SO_TIMESTAMPING: 2130 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2131 return -EINVAL; 2132 2133 tsflags = *(u32 *)CMSG_DATA(cmsg); 2134 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 2135 return -EINVAL; 2136 2137 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 2138 sockc->tsflags |= tsflags; 2139 break; 2140 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 2141 case SCM_RIGHTS: 2142 case SCM_CREDENTIALS: 2143 break; 2144 default: 2145 return -EINVAL; 2146 } 2147 return 0; 2148 } 2149 EXPORT_SYMBOL(__sock_cmsg_send); 2150 2151 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 2152 struct sockcm_cookie *sockc) 2153 { 2154 struct cmsghdr *cmsg; 2155 int ret; 2156 2157 for_each_cmsghdr(cmsg, msg) { 2158 if (!CMSG_OK(msg, cmsg)) 2159 return -EINVAL; 2160 if (cmsg->cmsg_level != SOL_SOCKET) 2161 continue; 2162 ret = __sock_cmsg_send(sk, msg, cmsg, sockc); 2163 if (ret) 2164 return ret; 2165 } 2166 return 0; 2167 } 2168 EXPORT_SYMBOL(sock_cmsg_send); 2169 2170 static void sk_enter_memory_pressure(struct sock *sk) 2171 { 2172 if (!sk->sk_prot->enter_memory_pressure) 2173 return; 2174 2175 sk->sk_prot->enter_memory_pressure(sk); 2176 } 2177 2178 static void sk_leave_memory_pressure(struct sock *sk) 2179 { 2180 if (sk->sk_prot->leave_memory_pressure) { 2181 sk->sk_prot->leave_memory_pressure(sk); 2182 } else { 2183 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 2184 2185 if (memory_pressure && *memory_pressure) 2186 *memory_pressure = 0; 2187 } 2188 } 2189 2190 /* On 32bit arches, an skb frag is limited to 2^15 */ 2191 #define SKB_FRAG_PAGE_ORDER get_order(32768) 2192 2193 /** 2194 * skb_page_frag_refill - check that a page_frag contains enough room 2195 * @sz: minimum size of the fragment we want to get 2196 * @pfrag: pointer to page_frag 2197 * @gfp: priority for memory allocation 2198 * 2199 * Note: While this allocator tries to use high order pages, there is 2200 * no guarantee that allocations succeed. Therefore, @sz MUST be 2201 * less or equal than PAGE_SIZE. 2202 */ 2203 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 2204 { 2205 if (pfrag->page) { 2206 if (page_ref_count(pfrag->page) == 1) { 2207 pfrag->offset = 0; 2208 return true; 2209 } 2210 if (pfrag->offset + sz <= pfrag->size) 2211 return true; 2212 put_page(pfrag->page); 2213 } 2214 2215 pfrag->offset = 0; 2216 if (SKB_FRAG_PAGE_ORDER) { 2217 /* Avoid direct reclaim but allow kswapd to wake */ 2218 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 2219 __GFP_COMP | __GFP_NOWARN | 2220 __GFP_NORETRY, 2221 SKB_FRAG_PAGE_ORDER); 2222 if (likely(pfrag->page)) { 2223 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 2224 return true; 2225 } 2226 } 2227 pfrag->page = alloc_page(gfp); 2228 if (likely(pfrag->page)) { 2229 pfrag->size = PAGE_SIZE; 2230 return true; 2231 } 2232 return false; 2233 } 2234 EXPORT_SYMBOL(skb_page_frag_refill); 2235 2236 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2237 { 2238 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2239 return true; 2240 2241 sk_enter_memory_pressure(sk); 2242 sk_stream_moderate_sndbuf(sk); 2243 return false; 2244 } 2245 EXPORT_SYMBOL(sk_page_frag_refill); 2246 2247 int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg, 2248 int sg_start, int *sg_curr_index, unsigned int *sg_curr_size, 2249 int first_coalesce) 2250 { 2251 int sg_curr = *sg_curr_index, use = 0, rc = 0; 2252 unsigned int size = *sg_curr_size; 2253 struct page_frag *pfrag; 2254 struct scatterlist *sge; 2255 2256 len -= size; 2257 pfrag = sk_page_frag(sk); 2258 2259 while (len > 0) { 2260 unsigned int orig_offset; 2261 2262 if (!sk_page_frag_refill(sk, pfrag)) { 2263 rc = -ENOMEM; 2264 goto out; 2265 } 2266 2267 use = min_t(int, len, pfrag->size - pfrag->offset); 2268 2269 if (!sk_wmem_schedule(sk, use)) { 2270 rc = -ENOMEM; 2271 goto out; 2272 } 2273 2274 sk_mem_charge(sk, use); 2275 size += use; 2276 orig_offset = pfrag->offset; 2277 pfrag->offset += use; 2278 2279 sge = sg + sg_curr - 1; 2280 if (sg_curr > first_coalesce && sg_page(sg) == pfrag->page && 2281 sg->offset + sg->length == orig_offset) { 2282 sg->length += use; 2283 } else { 2284 sge = sg + sg_curr; 2285 sg_unmark_end(sge); 2286 sg_set_page(sge, pfrag->page, use, orig_offset); 2287 get_page(pfrag->page); 2288 sg_curr++; 2289 2290 if (sg_curr == MAX_SKB_FRAGS) 2291 sg_curr = 0; 2292 2293 if (sg_curr == sg_start) { 2294 rc = -ENOSPC; 2295 break; 2296 } 2297 } 2298 2299 len -= use; 2300 } 2301 out: 2302 *sg_curr_size = size; 2303 *sg_curr_index = sg_curr; 2304 return rc; 2305 } 2306 EXPORT_SYMBOL(sk_alloc_sg); 2307 2308 static void __lock_sock(struct sock *sk) 2309 __releases(&sk->sk_lock.slock) 2310 __acquires(&sk->sk_lock.slock) 2311 { 2312 DEFINE_WAIT(wait); 2313 2314 for (;;) { 2315 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2316 TASK_UNINTERRUPTIBLE); 2317 spin_unlock_bh(&sk->sk_lock.slock); 2318 schedule(); 2319 spin_lock_bh(&sk->sk_lock.slock); 2320 if (!sock_owned_by_user(sk)) 2321 break; 2322 } 2323 finish_wait(&sk->sk_lock.wq, &wait); 2324 } 2325 2326 static void __release_sock(struct sock *sk) 2327 __releases(&sk->sk_lock.slock) 2328 __acquires(&sk->sk_lock.slock) 2329 { 2330 struct sk_buff *skb, *next; 2331 2332 while ((skb = sk->sk_backlog.head) != NULL) { 2333 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2334 2335 spin_unlock_bh(&sk->sk_lock.slock); 2336 2337 do { 2338 next = skb->next; 2339 prefetch(next); 2340 WARN_ON_ONCE(skb_dst_is_noref(skb)); 2341 skb->next = NULL; 2342 sk_backlog_rcv(sk, skb); 2343 2344 cond_resched(); 2345 2346 skb = next; 2347 } while (skb != NULL); 2348 2349 spin_lock_bh(&sk->sk_lock.slock); 2350 } 2351 2352 /* 2353 * Doing the zeroing here guarantee we can not loop forever 2354 * while a wild producer attempts to flood us. 2355 */ 2356 sk->sk_backlog.len = 0; 2357 } 2358 2359 void __sk_flush_backlog(struct sock *sk) 2360 { 2361 spin_lock_bh(&sk->sk_lock.slock); 2362 __release_sock(sk); 2363 spin_unlock_bh(&sk->sk_lock.slock); 2364 } 2365 2366 /** 2367 * sk_wait_data - wait for data to arrive at sk_receive_queue 2368 * @sk: sock to wait on 2369 * @timeo: for how long 2370 * @skb: last skb seen on sk_receive_queue 2371 * 2372 * Now socket state including sk->sk_err is changed only under lock, 2373 * hence we may omit checks after joining wait queue. 2374 * We check receive queue before schedule() only as optimization; 2375 * it is very likely that release_sock() added new data. 2376 */ 2377 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 2378 { 2379 DEFINE_WAIT_FUNC(wait, woken_wake_function); 2380 int rc; 2381 2382 add_wait_queue(sk_sleep(sk), &wait); 2383 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2384 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 2385 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2386 remove_wait_queue(sk_sleep(sk), &wait); 2387 return rc; 2388 } 2389 EXPORT_SYMBOL(sk_wait_data); 2390 2391 /** 2392 * __sk_mem_raise_allocated - increase memory_allocated 2393 * @sk: socket 2394 * @size: memory size to allocate 2395 * @amt: pages to allocate 2396 * @kind: allocation type 2397 * 2398 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc 2399 */ 2400 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 2401 { 2402 struct proto *prot = sk->sk_prot; 2403 long allocated = sk_memory_allocated_add(sk, amt); 2404 2405 if (mem_cgroup_sockets_enabled && sk->sk_memcg && 2406 !mem_cgroup_charge_skmem(sk->sk_memcg, amt)) 2407 goto suppress_allocation; 2408 2409 /* Under limit. */ 2410 if (allocated <= sk_prot_mem_limits(sk, 0)) { 2411 sk_leave_memory_pressure(sk); 2412 return 1; 2413 } 2414 2415 /* Under pressure. */ 2416 if (allocated > sk_prot_mem_limits(sk, 1)) 2417 sk_enter_memory_pressure(sk); 2418 2419 /* Over hard limit. */ 2420 if (allocated > sk_prot_mem_limits(sk, 2)) 2421 goto suppress_allocation; 2422 2423 /* guarantee minimum buffer size under pressure */ 2424 if (kind == SK_MEM_RECV) { 2425 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 2426 return 1; 2427 2428 } else { /* SK_MEM_SEND */ 2429 int wmem0 = sk_get_wmem0(sk, prot); 2430 2431 if (sk->sk_type == SOCK_STREAM) { 2432 if (sk->sk_wmem_queued < wmem0) 2433 return 1; 2434 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 2435 return 1; 2436 } 2437 } 2438 2439 if (sk_has_memory_pressure(sk)) { 2440 int alloc; 2441 2442 if (!sk_under_memory_pressure(sk)) 2443 return 1; 2444 alloc = sk_sockets_allocated_read_positive(sk); 2445 if (sk_prot_mem_limits(sk, 2) > alloc * 2446 sk_mem_pages(sk->sk_wmem_queued + 2447 atomic_read(&sk->sk_rmem_alloc) + 2448 sk->sk_forward_alloc)) 2449 return 1; 2450 } 2451 2452 suppress_allocation: 2453 2454 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 2455 sk_stream_moderate_sndbuf(sk); 2456 2457 /* Fail only if socket is _under_ its sndbuf. 2458 * In this case we cannot block, so that we have to fail. 2459 */ 2460 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) 2461 return 1; 2462 } 2463 2464 trace_sock_exceed_buf_limit(sk, prot, allocated); 2465 2466 sk_memory_allocated_sub(sk, amt); 2467 2468 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2469 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt); 2470 2471 return 0; 2472 } 2473 EXPORT_SYMBOL(__sk_mem_raise_allocated); 2474 2475 /** 2476 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 2477 * @sk: socket 2478 * @size: memory size to allocate 2479 * @kind: allocation type 2480 * 2481 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 2482 * rmem allocation. This function assumes that protocols which have 2483 * memory_pressure use sk_wmem_queued as write buffer accounting. 2484 */ 2485 int __sk_mem_schedule(struct sock *sk, int size, int kind) 2486 { 2487 int ret, amt = sk_mem_pages(size); 2488 2489 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT; 2490 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 2491 if (!ret) 2492 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT; 2493 return ret; 2494 } 2495 EXPORT_SYMBOL(__sk_mem_schedule); 2496 2497 /** 2498 * __sk_mem_reduce_allocated - reclaim memory_allocated 2499 * @sk: socket 2500 * @amount: number of quanta 2501 * 2502 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 2503 */ 2504 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 2505 { 2506 sk_memory_allocated_sub(sk, amount); 2507 2508 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2509 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 2510 2511 if (sk_under_memory_pressure(sk) && 2512 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 2513 sk_leave_memory_pressure(sk); 2514 } 2515 EXPORT_SYMBOL(__sk_mem_reduce_allocated); 2516 2517 /** 2518 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 2519 * @sk: socket 2520 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple) 2521 */ 2522 void __sk_mem_reclaim(struct sock *sk, int amount) 2523 { 2524 amount >>= SK_MEM_QUANTUM_SHIFT; 2525 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT; 2526 __sk_mem_reduce_allocated(sk, amount); 2527 } 2528 EXPORT_SYMBOL(__sk_mem_reclaim); 2529 2530 int sk_set_peek_off(struct sock *sk, int val) 2531 { 2532 sk->sk_peek_off = val; 2533 return 0; 2534 } 2535 EXPORT_SYMBOL_GPL(sk_set_peek_off); 2536 2537 /* 2538 * Set of default routines for initialising struct proto_ops when 2539 * the protocol does not support a particular function. In certain 2540 * cases where it makes no sense for a protocol to have a "do nothing" 2541 * function, some default processing is provided. 2542 */ 2543 2544 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 2545 { 2546 return -EOPNOTSUPP; 2547 } 2548 EXPORT_SYMBOL(sock_no_bind); 2549 2550 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 2551 int len, int flags) 2552 { 2553 return -EOPNOTSUPP; 2554 } 2555 EXPORT_SYMBOL(sock_no_connect); 2556 2557 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 2558 { 2559 return -EOPNOTSUPP; 2560 } 2561 EXPORT_SYMBOL(sock_no_socketpair); 2562 2563 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, 2564 bool kern) 2565 { 2566 return -EOPNOTSUPP; 2567 } 2568 EXPORT_SYMBOL(sock_no_accept); 2569 2570 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 2571 int peer) 2572 { 2573 return -EOPNOTSUPP; 2574 } 2575 EXPORT_SYMBOL(sock_no_getname); 2576 2577 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 2578 { 2579 return -EOPNOTSUPP; 2580 } 2581 EXPORT_SYMBOL(sock_no_ioctl); 2582 2583 int sock_no_listen(struct socket *sock, int backlog) 2584 { 2585 return -EOPNOTSUPP; 2586 } 2587 EXPORT_SYMBOL(sock_no_listen); 2588 2589 int sock_no_shutdown(struct socket *sock, int how) 2590 { 2591 return -EOPNOTSUPP; 2592 } 2593 EXPORT_SYMBOL(sock_no_shutdown); 2594 2595 int sock_no_setsockopt(struct socket *sock, int level, int optname, 2596 char __user *optval, unsigned int optlen) 2597 { 2598 return -EOPNOTSUPP; 2599 } 2600 EXPORT_SYMBOL(sock_no_setsockopt); 2601 2602 int sock_no_getsockopt(struct socket *sock, int level, int optname, 2603 char __user *optval, int __user *optlen) 2604 { 2605 return -EOPNOTSUPP; 2606 } 2607 EXPORT_SYMBOL(sock_no_getsockopt); 2608 2609 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 2610 { 2611 return -EOPNOTSUPP; 2612 } 2613 EXPORT_SYMBOL(sock_no_sendmsg); 2614 2615 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 2616 { 2617 return -EOPNOTSUPP; 2618 } 2619 EXPORT_SYMBOL(sock_no_sendmsg_locked); 2620 2621 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 2622 int flags) 2623 { 2624 return -EOPNOTSUPP; 2625 } 2626 EXPORT_SYMBOL(sock_no_recvmsg); 2627 2628 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 2629 { 2630 /* Mirror missing mmap method error code */ 2631 return -ENODEV; 2632 } 2633 EXPORT_SYMBOL(sock_no_mmap); 2634 2635 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) 2636 { 2637 ssize_t res; 2638 struct msghdr msg = {.msg_flags = flags}; 2639 struct kvec iov; 2640 char *kaddr = kmap(page); 2641 iov.iov_base = kaddr + offset; 2642 iov.iov_len = size; 2643 res = kernel_sendmsg(sock, &msg, &iov, 1, size); 2644 kunmap(page); 2645 return res; 2646 } 2647 EXPORT_SYMBOL(sock_no_sendpage); 2648 2649 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page, 2650 int offset, size_t size, int flags) 2651 { 2652 ssize_t res; 2653 struct msghdr msg = {.msg_flags = flags}; 2654 struct kvec iov; 2655 char *kaddr = kmap(page); 2656 2657 iov.iov_base = kaddr + offset; 2658 iov.iov_len = size; 2659 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size); 2660 kunmap(page); 2661 return res; 2662 } 2663 EXPORT_SYMBOL(sock_no_sendpage_locked); 2664 2665 /* 2666 * Default Socket Callbacks 2667 */ 2668 2669 static void sock_def_wakeup(struct sock *sk) 2670 { 2671 struct socket_wq *wq; 2672 2673 rcu_read_lock(); 2674 wq = rcu_dereference(sk->sk_wq); 2675 if (skwq_has_sleeper(wq)) 2676 wake_up_interruptible_all(&wq->wait); 2677 rcu_read_unlock(); 2678 } 2679 2680 static void sock_def_error_report(struct sock *sk) 2681 { 2682 struct socket_wq *wq; 2683 2684 rcu_read_lock(); 2685 wq = rcu_dereference(sk->sk_wq); 2686 if (skwq_has_sleeper(wq)) 2687 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 2688 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 2689 rcu_read_unlock(); 2690 } 2691 2692 static void sock_def_readable(struct sock *sk) 2693 { 2694 struct socket_wq *wq; 2695 2696 rcu_read_lock(); 2697 wq = rcu_dereference(sk->sk_wq); 2698 if (skwq_has_sleeper(wq)) 2699 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 2700 EPOLLRDNORM | EPOLLRDBAND); 2701 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 2702 rcu_read_unlock(); 2703 } 2704 2705 static void sock_def_write_space(struct sock *sk) 2706 { 2707 struct socket_wq *wq; 2708 2709 rcu_read_lock(); 2710 2711 /* Do not wake up a writer until he can make "significant" 2712 * progress. --DaveM 2713 */ 2714 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) { 2715 wq = rcu_dereference(sk->sk_wq); 2716 if (skwq_has_sleeper(wq)) 2717 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 2718 EPOLLWRNORM | EPOLLWRBAND); 2719 2720 /* Should agree with poll, otherwise some programs break */ 2721 if (sock_writeable(sk)) 2722 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 2723 } 2724 2725 rcu_read_unlock(); 2726 } 2727 2728 static void sock_def_destruct(struct sock *sk) 2729 { 2730 } 2731 2732 void sk_send_sigurg(struct sock *sk) 2733 { 2734 if (sk->sk_socket && sk->sk_socket->file) 2735 if (send_sigurg(&sk->sk_socket->file->f_owner)) 2736 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 2737 } 2738 EXPORT_SYMBOL(sk_send_sigurg); 2739 2740 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 2741 unsigned long expires) 2742 { 2743 if (!mod_timer(timer, expires)) 2744 sock_hold(sk); 2745 } 2746 EXPORT_SYMBOL(sk_reset_timer); 2747 2748 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 2749 { 2750 if (del_timer(timer)) 2751 __sock_put(sk); 2752 } 2753 EXPORT_SYMBOL(sk_stop_timer); 2754 2755 void sock_init_data(struct socket *sock, struct sock *sk) 2756 { 2757 sk_init_common(sk); 2758 sk->sk_send_head = NULL; 2759 2760 timer_setup(&sk->sk_timer, NULL, 0); 2761 2762 sk->sk_allocation = GFP_KERNEL; 2763 sk->sk_rcvbuf = sysctl_rmem_default; 2764 sk->sk_sndbuf = sysctl_wmem_default; 2765 sk->sk_state = TCP_CLOSE; 2766 sk_set_socket(sk, sock); 2767 2768 sock_set_flag(sk, SOCK_ZAPPED); 2769 2770 if (sock) { 2771 sk->sk_type = sock->type; 2772 sk->sk_wq = sock->wq; 2773 sock->sk = sk; 2774 sk->sk_uid = SOCK_INODE(sock)->i_uid; 2775 } else { 2776 sk->sk_wq = NULL; 2777 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0); 2778 } 2779 2780 rwlock_init(&sk->sk_callback_lock); 2781 if (sk->sk_kern_sock) 2782 lockdep_set_class_and_name( 2783 &sk->sk_callback_lock, 2784 af_kern_callback_keys + sk->sk_family, 2785 af_family_kern_clock_key_strings[sk->sk_family]); 2786 else 2787 lockdep_set_class_and_name( 2788 &sk->sk_callback_lock, 2789 af_callback_keys + sk->sk_family, 2790 af_family_clock_key_strings[sk->sk_family]); 2791 2792 sk->sk_state_change = sock_def_wakeup; 2793 sk->sk_data_ready = sock_def_readable; 2794 sk->sk_write_space = sock_def_write_space; 2795 sk->sk_error_report = sock_def_error_report; 2796 sk->sk_destruct = sock_def_destruct; 2797 2798 sk->sk_frag.page = NULL; 2799 sk->sk_frag.offset = 0; 2800 sk->sk_peek_off = -1; 2801 2802 sk->sk_peer_pid = NULL; 2803 sk->sk_peer_cred = NULL; 2804 sk->sk_write_pending = 0; 2805 sk->sk_rcvlowat = 1; 2806 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 2807 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 2808 2809 sk->sk_stamp = SK_DEFAULT_STAMP; 2810 atomic_set(&sk->sk_zckey, 0); 2811 2812 #ifdef CONFIG_NET_RX_BUSY_POLL 2813 sk->sk_napi_id = 0; 2814 sk->sk_ll_usec = sysctl_net_busy_read; 2815 #endif 2816 2817 sk->sk_max_pacing_rate = ~0U; 2818 sk->sk_pacing_rate = ~0U; 2819 sk->sk_pacing_shift = 10; 2820 sk->sk_incoming_cpu = -1; 2821 /* 2822 * Before updating sk_refcnt, we must commit prior changes to memory 2823 * (Documentation/RCU/rculist_nulls.txt for details) 2824 */ 2825 smp_wmb(); 2826 refcount_set(&sk->sk_refcnt, 1); 2827 atomic_set(&sk->sk_drops, 0); 2828 } 2829 EXPORT_SYMBOL(sock_init_data); 2830 2831 void lock_sock_nested(struct sock *sk, int subclass) 2832 { 2833 might_sleep(); 2834 spin_lock_bh(&sk->sk_lock.slock); 2835 if (sk->sk_lock.owned) 2836 __lock_sock(sk); 2837 sk->sk_lock.owned = 1; 2838 spin_unlock(&sk->sk_lock.slock); 2839 /* 2840 * The sk_lock has mutex_lock() semantics here: 2841 */ 2842 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 2843 local_bh_enable(); 2844 } 2845 EXPORT_SYMBOL(lock_sock_nested); 2846 2847 void release_sock(struct sock *sk) 2848 { 2849 spin_lock_bh(&sk->sk_lock.slock); 2850 if (sk->sk_backlog.tail) 2851 __release_sock(sk); 2852 2853 /* Warning : release_cb() might need to release sk ownership, 2854 * ie call sock_release_ownership(sk) before us. 2855 */ 2856 if (sk->sk_prot->release_cb) 2857 sk->sk_prot->release_cb(sk); 2858 2859 sock_release_ownership(sk); 2860 if (waitqueue_active(&sk->sk_lock.wq)) 2861 wake_up(&sk->sk_lock.wq); 2862 spin_unlock_bh(&sk->sk_lock.slock); 2863 } 2864 EXPORT_SYMBOL(release_sock); 2865 2866 /** 2867 * lock_sock_fast - fast version of lock_sock 2868 * @sk: socket 2869 * 2870 * This version should be used for very small section, where process wont block 2871 * return false if fast path is taken: 2872 * 2873 * sk_lock.slock locked, owned = 0, BH disabled 2874 * 2875 * return true if slow path is taken: 2876 * 2877 * sk_lock.slock unlocked, owned = 1, BH enabled 2878 */ 2879 bool lock_sock_fast(struct sock *sk) 2880 { 2881 might_sleep(); 2882 spin_lock_bh(&sk->sk_lock.slock); 2883 2884 if (!sk->sk_lock.owned) 2885 /* 2886 * Note : We must disable BH 2887 */ 2888 return false; 2889 2890 __lock_sock(sk); 2891 sk->sk_lock.owned = 1; 2892 spin_unlock(&sk->sk_lock.slock); 2893 /* 2894 * The sk_lock has mutex_lock() semantics here: 2895 */ 2896 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_); 2897 local_bh_enable(); 2898 return true; 2899 } 2900 EXPORT_SYMBOL(lock_sock_fast); 2901 2902 int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp) 2903 { 2904 struct timeval tv; 2905 if (!sock_flag(sk, SOCK_TIMESTAMP)) 2906 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2907 tv = ktime_to_timeval(sk->sk_stamp); 2908 if (tv.tv_sec == -1) 2909 return -ENOENT; 2910 if (tv.tv_sec == 0) { 2911 sk->sk_stamp = ktime_get_real(); 2912 tv = ktime_to_timeval(sk->sk_stamp); 2913 } 2914 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0; 2915 } 2916 EXPORT_SYMBOL(sock_get_timestamp); 2917 2918 int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp) 2919 { 2920 struct timespec ts; 2921 if (!sock_flag(sk, SOCK_TIMESTAMP)) 2922 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2923 ts = ktime_to_timespec(sk->sk_stamp); 2924 if (ts.tv_sec == -1) 2925 return -ENOENT; 2926 if (ts.tv_sec == 0) { 2927 sk->sk_stamp = ktime_get_real(); 2928 ts = ktime_to_timespec(sk->sk_stamp); 2929 } 2930 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0; 2931 } 2932 EXPORT_SYMBOL(sock_get_timestampns); 2933 2934 void sock_enable_timestamp(struct sock *sk, int flag) 2935 { 2936 if (!sock_flag(sk, flag)) { 2937 unsigned long previous_flags = sk->sk_flags; 2938 2939 sock_set_flag(sk, flag); 2940 /* 2941 * we just set one of the two flags which require net 2942 * time stamping, but time stamping might have been on 2943 * already because of the other one 2944 */ 2945 if (sock_needs_netstamp(sk) && 2946 !(previous_flags & SK_FLAGS_TIMESTAMP)) 2947 net_enable_timestamp(); 2948 } 2949 } 2950 2951 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 2952 int level, int type) 2953 { 2954 struct sock_exterr_skb *serr; 2955 struct sk_buff *skb; 2956 int copied, err; 2957 2958 err = -EAGAIN; 2959 skb = sock_dequeue_err_skb(sk); 2960 if (skb == NULL) 2961 goto out; 2962 2963 copied = skb->len; 2964 if (copied > len) { 2965 msg->msg_flags |= MSG_TRUNC; 2966 copied = len; 2967 } 2968 err = skb_copy_datagram_msg(skb, 0, msg, copied); 2969 if (err) 2970 goto out_free_skb; 2971 2972 sock_recv_timestamp(msg, sk, skb); 2973 2974 serr = SKB_EXT_ERR(skb); 2975 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 2976 2977 msg->msg_flags |= MSG_ERRQUEUE; 2978 err = copied; 2979 2980 out_free_skb: 2981 kfree_skb(skb); 2982 out: 2983 return err; 2984 } 2985 EXPORT_SYMBOL(sock_recv_errqueue); 2986 2987 /* 2988 * Get a socket option on an socket. 2989 * 2990 * FIX: POSIX 1003.1g is very ambiguous here. It states that 2991 * asynchronous errors should be reported by getsockopt. We assume 2992 * this means if you specify SO_ERROR (otherwise whats the point of it). 2993 */ 2994 int sock_common_getsockopt(struct socket *sock, int level, int optname, 2995 char __user *optval, int __user *optlen) 2996 { 2997 struct sock *sk = sock->sk; 2998 2999 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 3000 } 3001 EXPORT_SYMBOL(sock_common_getsockopt); 3002 3003 #ifdef CONFIG_COMPAT 3004 int compat_sock_common_getsockopt(struct socket *sock, int level, int optname, 3005 char __user *optval, int __user *optlen) 3006 { 3007 struct sock *sk = sock->sk; 3008 3009 if (sk->sk_prot->compat_getsockopt != NULL) 3010 return sk->sk_prot->compat_getsockopt(sk, level, optname, 3011 optval, optlen); 3012 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 3013 } 3014 EXPORT_SYMBOL(compat_sock_common_getsockopt); 3015 #endif 3016 3017 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 3018 int flags) 3019 { 3020 struct sock *sk = sock->sk; 3021 int addr_len = 0; 3022 int err; 3023 3024 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, 3025 flags & ~MSG_DONTWAIT, &addr_len); 3026 if (err >= 0) 3027 msg->msg_namelen = addr_len; 3028 return err; 3029 } 3030 EXPORT_SYMBOL(sock_common_recvmsg); 3031 3032 /* 3033 * Set socket options on an inet socket. 3034 */ 3035 int sock_common_setsockopt(struct socket *sock, int level, int optname, 3036 char __user *optval, unsigned int optlen) 3037 { 3038 struct sock *sk = sock->sk; 3039 3040 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3041 } 3042 EXPORT_SYMBOL(sock_common_setsockopt); 3043 3044 #ifdef CONFIG_COMPAT 3045 int compat_sock_common_setsockopt(struct socket *sock, int level, int optname, 3046 char __user *optval, unsigned int optlen) 3047 { 3048 struct sock *sk = sock->sk; 3049 3050 if (sk->sk_prot->compat_setsockopt != NULL) 3051 return sk->sk_prot->compat_setsockopt(sk, level, optname, 3052 optval, optlen); 3053 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3054 } 3055 EXPORT_SYMBOL(compat_sock_common_setsockopt); 3056 #endif 3057 3058 void sk_common_release(struct sock *sk) 3059 { 3060 if (sk->sk_prot->destroy) 3061 sk->sk_prot->destroy(sk); 3062 3063 /* 3064 * Observation: when sock_common_release is called, processes have 3065 * no access to socket. But net still has. 3066 * Step one, detach it from networking: 3067 * 3068 * A. Remove from hash tables. 3069 */ 3070 3071 sk->sk_prot->unhash(sk); 3072 3073 /* 3074 * In this point socket cannot receive new packets, but it is possible 3075 * that some packets are in flight because some CPU runs receiver and 3076 * did hash table lookup before we unhashed socket. They will achieve 3077 * receive queue and will be purged by socket destructor. 3078 * 3079 * Also we still have packets pending on receive queue and probably, 3080 * our own packets waiting in device queues. sock_destroy will drain 3081 * receive queue, but transmitted packets will delay socket destruction 3082 * until the last reference will be released. 3083 */ 3084 3085 sock_orphan(sk); 3086 3087 xfrm_sk_free_policy(sk); 3088 3089 sk_refcnt_debug_release(sk); 3090 3091 sock_put(sk); 3092 } 3093 EXPORT_SYMBOL(sk_common_release); 3094 3095 void sk_get_meminfo(const struct sock *sk, u32 *mem) 3096 { 3097 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 3098 3099 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 3100 mem[SK_MEMINFO_RCVBUF] = sk->sk_rcvbuf; 3101 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 3102 mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf; 3103 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc; 3104 mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued; 3105 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 3106 mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len; 3107 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops); 3108 } 3109 3110 #ifdef CONFIG_PROC_FS 3111 #define PROTO_INUSE_NR 64 /* should be enough for the first time */ 3112 struct prot_inuse { 3113 int val[PROTO_INUSE_NR]; 3114 }; 3115 3116 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 3117 3118 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) 3119 { 3120 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); 3121 } 3122 EXPORT_SYMBOL_GPL(sock_prot_inuse_add); 3123 3124 int sock_prot_inuse_get(struct net *net, struct proto *prot) 3125 { 3126 int cpu, idx = prot->inuse_idx; 3127 int res = 0; 3128 3129 for_each_possible_cpu(cpu) 3130 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 3131 3132 return res >= 0 ? res : 0; 3133 } 3134 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 3135 3136 static void sock_inuse_add(struct net *net, int val) 3137 { 3138 this_cpu_add(*net->core.sock_inuse, val); 3139 } 3140 3141 int sock_inuse_get(struct net *net) 3142 { 3143 int cpu, res = 0; 3144 3145 for_each_possible_cpu(cpu) 3146 res += *per_cpu_ptr(net->core.sock_inuse, cpu); 3147 3148 return res; 3149 } 3150 3151 EXPORT_SYMBOL_GPL(sock_inuse_get); 3152 3153 static int __net_init sock_inuse_init_net(struct net *net) 3154 { 3155 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 3156 if (net->core.prot_inuse == NULL) 3157 return -ENOMEM; 3158 3159 net->core.sock_inuse = alloc_percpu(int); 3160 if (net->core.sock_inuse == NULL) 3161 goto out; 3162 3163 return 0; 3164 3165 out: 3166 free_percpu(net->core.prot_inuse); 3167 return -ENOMEM; 3168 } 3169 3170 static void __net_exit sock_inuse_exit_net(struct net *net) 3171 { 3172 free_percpu(net->core.prot_inuse); 3173 free_percpu(net->core.sock_inuse); 3174 } 3175 3176 static struct pernet_operations net_inuse_ops = { 3177 .init = sock_inuse_init_net, 3178 .exit = sock_inuse_exit_net, 3179 }; 3180 3181 static __init int net_inuse_init(void) 3182 { 3183 if (register_pernet_subsys(&net_inuse_ops)) 3184 panic("Cannot initialize net inuse counters"); 3185 3186 return 0; 3187 } 3188 3189 core_initcall(net_inuse_init); 3190 3191 static void assign_proto_idx(struct proto *prot) 3192 { 3193 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 3194 3195 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 3196 pr_err("PROTO_INUSE_NR exhausted\n"); 3197 return; 3198 } 3199 3200 set_bit(prot->inuse_idx, proto_inuse_idx); 3201 } 3202 3203 static void release_proto_idx(struct proto *prot) 3204 { 3205 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 3206 clear_bit(prot->inuse_idx, proto_inuse_idx); 3207 } 3208 #else 3209 static inline void assign_proto_idx(struct proto *prot) 3210 { 3211 } 3212 3213 static inline void release_proto_idx(struct proto *prot) 3214 { 3215 } 3216 3217 static void sock_inuse_add(struct net *net, int val) 3218 { 3219 } 3220 #endif 3221 3222 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 3223 { 3224 if (!rsk_prot) 3225 return; 3226 kfree(rsk_prot->slab_name); 3227 rsk_prot->slab_name = NULL; 3228 kmem_cache_destroy(rsk_prot->slab); 3229 rsk_prot->slab = NULL; 3230 } 3231 3232 static int req_prot_init(const struct proto *prot) 3233 { 3234 struct request_sock_ops *rsk_prot = prot->rsk_prot; 3235 3236 if (!rsk_prot) 3237 return 0; 3238 3239 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 3240 prot->name); 3241 if (!rsk_prot->slab_name) 3242 return -ENOMEM; 3243 3244 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 3245 rsk_prot->obj_size, 0, 3246 SLAB_ACCOUNT | prot->slab_flags, 3247 NULL); 3248 3249 if (!rsk_prot->slab) { 3250 pr_crit("%s: Can't create request sock SLAB cache!\n", 3251 prot->name); 3252 return -ENOMEM; 3253 } 3254 return 0; 3255 } 3256 3257 int proto_register(struct proto *prot, int alloc_slab) 3258 { 3259 if (alloc_slab) { 3260 prot->slab = kmem_cache_create_usercopy(prot->name, 3261 prot->obj_size, 0, 3262 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 3263 prot->slab_flags, 3264 prot->useroffset, prot->usersize, 3265 NULL); 3266 3267 if (prot->slab == NULL) { 3268 pr_crit("%s: Can't create sock SLAB cache!\n", 3269 prot->name); 3270 goto out; 3271 } 3272 3273 if (req_prot_init(prot)) 3274 goto out_free_request_sock_slab; 3275 3276 if (prot->twsk_prot != NULL) { 3277 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name); 3278 3279 if (prot->twsk_prot->twsk_slab_name == NULL) 3280 goto out_free_request_sock_slab; 3281 3282 prot->twsk_prot->twsk_slab = 3283 kmem_cache_create(prot->twsk_prot->twsk_slab_name, 3284 prot->twsk_prot->twsk_obj_size, 3285 0, 3286 SLAB_ACCOUNT | 3287 prot->slab_flags, 3288 NULL); 3289 if (prot->twsk_prot->twsk_slab == NULL) 3290 goto out_free_timewait_sock_slab_name; 3291 } 3292 } 3293 3294 mutex_lock(&proto_list_mutex); 3295 list_add(&prot->node, &proto_list); 3296 assign_proto_idx(prot); 3297 mutex_unlock(&proto_list_mutex); 3298 return 0; 3299 3300 out_free_timewait_sock_slab_name: 3301 kfree(prot->twsk_prot->twsk_slab_name); 3302 out_free_request_sock_slab: 3303 req_prot_cleanup(prot->rsk_prot); 3304 3305 kmem_cache_destroy(prot->slab); 3306 prot->slab = NULL; 3307 out: 3308 return -ENOBUFS; 3309 } 3310 EXPORT_SYMBOL(proto_register); 3311 3312 void proto_unregister(struct proto *prot) 3313 { 3314 mutex_lock(&proto_list_mutex); 3315 release_proto_idx(prot); 3316 list_del(&prot->node); 3317 mutex_unlock(&proto_list_mutex); 3318 3319 kmem_cache_destroy(prot->slab); 3320 prot->slab = NULL; 3321 3322 req_prot_cleanup(prot->rsk_prot); 3323 3324 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) { 3325 kmem_cache_destroy(prot->twsk_prot->twsk_slab); 3326 kfree(prot->twsk_prot->twsk_slab_name); 3327 prot->twsk_prot->twsk_slab = NULL; 3328 } 3329 } 3330 EXPORT_SYMBOL(proto_unregister); 3331 3332 int sock_load_diag_module(int family, int protocol) 3333 { 3334 if (!protocol) { 3335 if (!sock_is_registered(family)) 3336 return -ENOENT; 3337 3338 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 3339 NETLINK_SOCK_DIAG, family); 3340 } 3341 3342 #ifdef CONFIG_INET 3343 if (family == AF_INET && 3344 !rcu_access_pointer(inet_protos[protocol])) 3345 return -ENOENT; 3346 #endif 3347 3348 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 3349 NETLINK_SOCK_DIAG, family, protocol); 3350 } 3351 EXPORT_SYMBOL(sock_load_diag_module); 3352 3353 #ifdef CONFIG_PROC_FS 3354 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 3355 __acquires(proto_list_mutex) 3356 { 3357 mutex_lock(&proto_list_mutex); 3358 return seq_list_start_head(&proto_list, *pos); 3359 } 3360 3361 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3362 { 3363 return seq_list_next(v, &proto_list, pos); 3364 } 3365 3366 static void proto_seq_stop(struct seq_file *seq, void *v) 3367 __releases(proto_list_mutex) 3368 { 3369 mutex_unlock(&proto_list_mutex); 3370 } 3371 3372 static char proto_method_implemented(const void *method) 3373 { 3374 return method == NULL ? 'n' : 'y'; 3375 } 3376 static long sock_prot_memory_allocated(struct proto *proto) 3377 { 3378 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 3379 } 3380 3381 static char *sock_prot_memory_pressure(struct proto *proto) 3382 { 3383 return proto->memory_pressure != NULL ? 3384 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 3385 } 3386 3387 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 3388 { 3389 3390 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 3391 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 3392 proto->name, 3393 proto->obj_size, 3394 sock_prot_inuse_get(seq_file_net(seq), proto), 3395 sock_prot_memory_allocated(proto), 3396 sock_prot_memory_pressure(proto), 3397 proto->max_header, 3398 proto->slab == NULL ? "no" : "yes", 3399 module_name(proto->owner), 3400 proto_method_implemented(proto->close), 3401 proto_method_implemented(proto->connect), 3402 proto_method_implemented(proto->disconnect), 3403 proto_method_implemented(proto->accept), 3404 proto_method_implemented(proto->ioctl), 3405 proto_method_implemented(proto->init), 3406 proto_method_implemented(proto->destroy), 3407 proto_method_implemented(proto->shutdown), 3408 proto_method_implemented(proto->setsockopt), 3409 proto_method_implemented(proto->getsockopt), 3410 proto_method_implemented(proto->sendmsg), 3411 proto_method_implemented(proto->recvmsg), 3412 proto_method_implemented(proto->sendpage), 3413 proto_method_implemented(proto->bind), 3414 proto_method_implemented(proto->backlog_rcv), 3415 proto_method_implemented(proto->hash), 3416 proto_method_implemented(proto->unhash), 3417 proto_method_implemented(proto->get_port), 3418 proto_method_implemented(proto->enter_memory_pressure)); 3419 } 3420 3421 static int proto_seq_show(struct seq_file *seq, void *v) 3422 { 3423 if (v == &proto_list) 3424 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 3425 "protocol", 3426 "size", 3427 "sockets", 3428 "memory", 3429 "press", 3430 "maxhdr", 3431 "slab", 3432 "module", 3433 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n"); 3434 else 3435 proto_seq_printf(seq, list_entry(v, struct proto, node)); 3436 return 0; 3437 } 3438 3439 static const struct seq_operations proto_seq_ops = { 3440 .start = proto_seq_start, 3441 .next = proto_seq_next, 3442 .stop = proto_seq_stop, 3443 .show = proto_seq_show, 3444 }; 3445 3446 static __net_init int proto_init_net(struct net *net) 3447 { 3448 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 3449 sizeof(struct seq_net_private))) 3450 return -ENOMEM; 3451 3452 return 0; 3453 } 3454 3455 static __net_exit void proto_exit_net(struct net *net) 3456 { 3457 remove_proc_entry("protocols", net->proc_net); 3458 } 3459 3460 3461 static __net_initdata struct pernet_operations proto_net_ops = { 3462 .init = proto_init_net, 3463 .exit = proto_exit_net, 3464 }; 3465 3466 static int __init proto_init(void) 3467 { 3468 return register_pernet_subsys(&proto_net_ops); 3469 } 3470 3471 subsys_initcall(proto_init); 3472 3473 #endif /* PROC_FS */ 3474 3475 #ifdef CONFIG_NET_RX_BUSY_POLL 3476 bool sk_busy_loop_end(void *p, unsigned long start_time) 3477 { 3478 struct sock *sk = p; 3479 3480 return !skb_queue_empty(&sk->sk_receive_queue) || 3481 sk_busy_loop_timeout(sk, start_time); 3482 } 3483 EXPORT_SYMBOL(sk_busy_loop_end); 3484 #endif /* CONFIG_NET_RX_BUSY_POLL */ 3485