1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Generic socket support routines. Memory allocators, socket lock/release 7 * handler for protocols to use and generic option handler. 8 * 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 * 85 * 86 * This program is free software; you can redistribute it and/or 87 * modify it under the terms of the GNU General Public License 88 * as published by the Free Software Foundation; either version 89 * 2 of the License, or (at your option) any later version. 90 */ 91 92 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 93 94 #include <asm/unaligned.h> 95 #include <linux/capability.h> 96 #include <linux/errno.h> 97 #include <linux/errqueue.h> 98 #include <linux/types.h> 99 #include <linux/socket.h> 100 #include <linux/in.h> 101 #include <linux/kernel.h> 102 #include <linux/module.h> 103 #include <linux/proc_fs.h> 104 #include <linux/seq_file.h> 105 #include <linux/sched.h> 106 #include <linux/sched/mm.h> 107 #include <linux/timer.h> 108 #include <linux/string.h> 109 #include <linux/sockios.h> 110 #include <linux/net.h> 111 #include <linux/mm.h> 112 #include <linux/slab.h> 113 #include <linux/interrupt.h> 114 #include <linux/poll.h> 115 #include <linux/tcp.h> 116 #include <linux/init.h> 117 #include <linux/highmem.h> 118 #include <linux/user_namespace.h> 119 #include <linux/static_key.h> 120 #include <linux/memcontrol.h> 121 #include <linux/prefetch.h> 122 123 #include <linux/uaccess.h> 124 125 #include <linux/netdevice.h> 126 #include <net/protocol.h> 127 #include <linux/skbuff.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <linux/net_tstamp.h> 132 #include <net/xfrm.h> 133 #include <linux/ipsec.h> 134 #include <net/cls_cgroup.h> 135 #include <net/netprio_cgroup.h> 136 #include <linux/sock_diag.h> 137 138 #include <linux/filter.h> 139 #include <net/sock_reuseport.h> 140 141 #include <trace/events/sock.h> 142 143 #include <net/tcp.h> 144 #include <net/busy_poll.h> 145 146 static DEFINE_MUTEX(proto_list_mutex); 147 static LIST_HEAD(proto_list); 148 149 static void sock_inuse_add(struct net *net, int val); 150 151 /** 152 * sk_ns_capable - General socket capability test 153 * @sk: Socket to use a capability on or through 154 * @user_ns: The user namespace of the capability to use 155 * @cap: The capability to use 156 * 157 * Test to see if the opener of the socket had when the socket was 158 * created and the current process has the capability @cap in the user 159 * namespace @user_ns. 160 */ 161 bool sk_ns_capable(const struct sock *sk, 162 struct user_namespace *user_ns, int cap) 163 { 164 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 165 ns_capable(user_ns, cap); 166 } 167 EXPORT_SYMBOL(sk_ns_capable); 168 169 /** 170 * sk_capable - Socket global capability test 171 * @sk: Socket to use a capability on or through 172 * @cap: The global capability to use 173 * 174 * Test to see if the opener of the socket had when the socket was 175 * created and the current process has the capability @cap in all user 176 * namespaces. 177 */ 178 bool sk_capable(const struct sock *sk, int cap) 179 { 180 return sk_ns_capable(sk, &init_user_ns, cap); 181 } 182 EXPORT_SYMBOL(sk_capable); 183 184 /** 185 * sk_net_capable - Network namespace socket capability test 186 * @sk: Socket to use a capability on or through 187 * @cap: The capability to use 188 * 189 * Test to see if the opener of the socket had when the socket was created 190 * and the current process has the capability @cap over the network namespace 191 * the socket is a member of. 192 */ 193 bool sk_net_capable(const struct sock *sk, int cap) 194 { 195 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 196 } 197 EXPORT_SYMBOL(sk_net_capable); 198 199 /* 200 * Each address family might have different locking rules, so we have 201 * one slock key per address family and separate keys for internal and 202 * userspace sockets. 203 */ 204 static struct lock_class_key af_family_keys[AF_MAX]; 205 static struct lock_class_key af_family_kern_keys[AF_MAX]; 206 static struct lock_class_key af_family_slock_keys[AF_MAX]; 207 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 208 209 /* 210 * Make lock validator output more readable. (we pre-construct these 211 * strings build-time, so that runtime initialization of socket 212 * locks is fast): 213 */ 214 215 #define _sock_locks(x) \ 216 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 217 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 218 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 219 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 220 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 221 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 222 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 223 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 224 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 225 x "27" , x "28" , x "AF_CAN" , \ 226 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 227 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 228 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 229 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 230 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 231 x "AF_MAX" 232 233 static const char *const af_family_key_strings[AF_MAX+1] = { 234 _sock_locks("sk_lock-") 235 }; 236 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 237 _sock_locks("slock-") 238 }; 239 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 240 _sock_locks("clock-") 241 }; 242 243 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 244 _sock_locks("k-sk_lock-") 245 }; 246 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 247 _sock_locks("k-slock-") 248 }; 249 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 250 _sock_locks("k-clock-") 251 }; 252 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 253 _sock_locks("rlock-") 254 }; 255 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 256 _sock_locks("wlock-") 257 }; 258 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 259 _sock_locks("elock-") 260 }; 261 262 /* 263 * sk_callback_lock and sk queues locking rules are per-address-family, 264 * so split the lock classes by using a per-AF key: 265 */ 266 static struct lock_class_key af_callback_keys[AF_MAX]; 267 static struct lock_class_key af_rlock_keys[AF_MAX]; 268 static struct lock_class_key af_wlock_keys[AF_MAX]; 269 static struct lock_class_key af_elock_keys[AF_MAX]; 270 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 271 272 /* Run time adjustable parameters. */ 273 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 274 EXPORT_SYMBOL(sysctl_wmem_max); 275 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 276 EXPORT_SYMBOL(sysctl_rmem_max); 277 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 278 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 279 280 /* Maximal space eaten by iovec or ancillary data plus some space */ 281 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 282 EXPORT_SYMBOL(sysctl_optmem_max); 283 284 int sysctl_tstamp_allow_data __read_mostly = 1; 285 286 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 287 EXPORT_SYMBOL_GPL(memalloc_socks_key); 288 289 /** 290 * sk_set_memalloc - sets %SOCK_MEMALLOC 291 * @sk: socket to set it on 292 * 293 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 294 * It's the responsibility of the admin to adjust min_free_kbytes 295 * to meet the requirements 296 */ 297 void sk_set_memalloc(struct sock *sk) 298 { 299 sock_set_flag(sk, SOCK_MEMALLOC); 300 sk->sk_allocation |= __GFP_MEMALLOC; 301 static_branch_inc(&memalloc_socks_key); 302 } 303 EXPORT_SYMBOL_GPL(sk_set_memalloc); 304 305 void sk_clear_memalloc(struct sock *sk) 306 { 307 sock_reset_flag(sk, SOCK_MEMALLOC); 308 sk->sk_allocation &= ~__GFP_MEMALLOC; 309 static_branch_dec(&memalloc_socks_key); 310 311 /* 312 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 313 * progress of swapping. SOCK_MEMALLOC may be cleared while 314 * it has rmem allocations due to the last swapfile being deactivated 315 * but there is a risk that the socket is unusable due to exceeding 316 * the rmem limits. Reclaim the reserves and obey rmem limits again. 317 */ 318 sk_mem_reclaim(sk); 319 } 320 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 321 322 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 323 { 324 int ret; 325 unsigned int noreclaim_flag; 326 327 /* these should have been dropped before queueing */ 328 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 329 330 noreclaim_flag = memalloc_noreclaim_save(); 331 ret = sk->sk_backlog_rcv(sk, skb); 332 memalloc_noreclaim_restore(noreclaim_flag); 333 334 return ret; 335 } 336 EXPORT_SYMBOL(__sk_backlog_rcv); 337 338 static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen) 339 { 340 struct timeval tv; 341 342 if (optlen < sizeof(tv)) 343 return -EINVAL; 344 if (copy_from_user(&tv, optval, sizeof(tv))) 345 return -EFAULT; 346 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 347 return -EDOM; 348 349 if (tv.tv_sec < 0) { 350 static int warned __read_mostly; 351 352 *timeo_p = 0; 353 if (warned < 10 && net_ratelimit()) { 354 warned++; 355 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 356 __func__, current->comm, task_pid_nr(current)); 357 } 358 return 0; 359 } 360 *timeo_p = MAX_SCHEDULE_TIMEOUT; 361 if (tv.tv_sec == 0 && tv.tv_usec == 0) 362 return 0; 363 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1)) 364 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ); 365 return 0; 366 } 367 368 static void sock_warn_obsolete_bsdism(const char *name) 369 { 370 static int warned; 371 static char warncomm[TASK_COMM_LEN]; 372 if (strcmp(warncomm, current->comm) && warned < 5) { 373 strcpy(warncomm, current->comm); 374 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n", 375 warncomm, name); 376 warned++; 377 } 378 } 379 380 static bool sock_needs_netstamp(const struct sock *sk) 381 { 382 switch (sk->sk_family) { 383 case AF_UNSPEC: 384 case AF_UNIX: 385 return false; 386 default: 387 return true; 388 } 389 } 390 391 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 392 { 393 if (sk->sk_flags & flags) { 394 sk->sk_flags &= ~flags; 395 if (sock_needs_netstamp(sk) && 396 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 397 net_disable_timestamp(); 398 } 399 } 400 401 402 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 403 { 404 unsigned long flags; 405 struct sk_buff_head *list = &sk->sk_receive_queue; 406 407 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { 408 atomic_inc(&sk->sk_drops); 409 trace_sock_rcvqueue_full(sk, skb); 410 return -ENOMEM; 411 } 412 413 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 414 atomic_inc(&sk->sk_drops); 415 return -ENOBUFS; 416 } 417 418 skb->dev = NULL; 419 skb_set_owner_r(skb, sk); 420 421 /* we escape from rcu protected region, make sure we dont leak 422 * a norefcounted dst 423 */ 424 skb_dst_force(skb); 425 426 spin_lock_irqsave(&list->lock, flags); 427 sock_skb_set_dropcount(sk, skb); 428 __skb_queue_tail(list, skb); 429 spin_unlock_irqrestore(&list->lock, flags); 430 431 if (!sock_flag(sk, SOCK_DEAD)) 432 sk->sk_data_ready(sk); 433 return 0; 434 } 435 EXPORT_SYMBOL(__sock_queue_rcv_skb); 436 437 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 438 { 439 int err; 440 441 err = sk_filter(sk, skb); 442 if (err) 443 return err; 444 445 return __sock_queue_rcv_skb(sk, skb); 446 } 447 EXPORT_SYMBOL(sock_queue_rcv_skb); 448 449 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 450 const int nested, unsigned int trim_cap, bool refcounted) 451 { 452 int rc = NET_RX_SUCCESS; 453 454 if (sk_filter_trim_cap(sk, skb, trim_cap)) 455 goto discard_and_relse; 456 457 skb->dev = NULL; 458 459 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) { 460 atomic_inc(&sk->sk_drops); 461 goto discard_and_relse; 462 } 463 if (nested) 464 bh_lock_sock_nested(sk); 465 else 466 bh_lock_sock(sk); 467 if (!sock_owned_by_user(sk)) { 468 /* 469 * trylock + unlock semantics: 470 */ 471 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 472 473 rc = sk_backlog_rcv(sk, skb); 474 475 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_); 476 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) { 477 bh_unlock_sock(sk); 478 atomic_inc(&sk->sk_drops); 479 goto discard_and_relse; 480 } 481 482 bh_unlock_sock(sk); 483 out: 484 if (refcounted) 485 sock_put(sk); 486 return rc; 487 discard_and_relse: 488 kfree_skb(skb); 489 goto out; 490 } 491 EXPORT_SYMBOL(__sk_receive_skb); 492 493 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 494 { 495 struct dst_entry *dst = __sk_dst_get(sk); 496 497 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 498 sk_tx_queue_clear(sk); 499 sk->sk_dst_pending_confirm = 0; 500 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 501 dst_release(dst); 502 return NULL; 503 } 504 505 return dst; 506 } 507 EXPORT_SYMBOL(__sk_dst_check); 508 509 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 510 { 511 struct dst_entry *dst = sk_dst_get(sk); 512 513 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 514 sk_dst_reset(sk); 515 dst_release(dst); 516 return NULL; 517 } 518 519 return dst; 520 } 521 EXPORT_SYMBOL(sk_dst_check); 522 523 static int sock_setbindtodevice(struct sock *sk, char __user *optval, 524 int optlen) 525 { 526 int ret = -ENOPROTOOPT; 527 #ifdef CONFIG_NETDEVICES 528 struct net *net = sock_net(sk); 529 char devname[IFNAMSIZ]; 530 int index; 531 532 /* Sorry... */ 533 ret = -EPERM; 534 if (!ns_capable(net->user_ns, CAP_NET_RAW)) 535 goto out; 536 537 ret = -EINVAL; 538 if (optlen < 0) 539 goto out; 540 541 /* Bind this socket to a particular device like "eth0", 542 * as specified in the passed interface name. If the 543 * name is "" or the option length is zero the socket 544 * is not bound. 545 */ 546 if (optlen > IFNAMSIZ - 1) 547 optlen = IFNAMSIZ - 1; 548 memset(devname, 0, sizeof(devname)); 549 550 ret = -EFAULT; 551 if (copy_from_user(devname, optval, optlen)) 552 goto out; 553 554 index = 0; 555 if (devname[0] != '\0') { 556 struct net_device *dev; 557 558 rcu_read_lock(); 559 dev = dev_get_by_name_rcu(net, devname); 560 if (dev) 561 index = dev->ifindex; 562 rcu_read_unlock(); 563 ret = -ENODEV; 564 if (!dev) 565 goto out; 566 } 567 568 lock_sock(sk); 569 sk->sk_bound_dev_if = index; 570 sk_dst_reset(sk); 571 release_sock(sk); 572 573 ret = 0; 574 575 out: 576 #endif 577 578 return ret; 579 } 580 581 static int sock_getbindtodevice(struct sock *sk, char __user *optval, 582 int __user *optlen, int len) 583 { 584 int ret = -ENOPROTOOPT; 585 #ifdef CONFIG_NETDEVICES 586 struct net *net = sock_net(sk); 587 char devname[IFNAMSIZ]; 588 589 if (sk->sk_bound_dev_if == 0) { 590 len = 0; 591 goto zero; 592 } 593 594 ret = -EINVAL; 595 if (len < IFNAMSIZ) 596 goto out; 597 598 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if); 599 if (ret) 600 goto out; 601 602 len = strlen(devname) + 1; 603 604 ret = -EFAULT; 605 if (copy_to_user(optval, devname, len)) 606 goto out; 607 608 zero: 609 ret = -EFAULT; 610 if (put_user(len, optlen)) 611 goto out; 612 613 ret = 0; 614 615 out: 616 #endif 617 618 return ret; 619 } 620 621 static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool) 622 { 623 if (valbool) 624 sock_set_flag(sk, bit); 625 else 626 sock_reset_flag(sk, bit); 627 } 628 629 bool sk_mc_loop(struct sock *sk) 630 { 631 if (dev_recursion_level()) 632 return false; 633 if (!sk) 634 return true; 635 switch (sk->sk_family) { 636 case AF_INET: 637 return inet_sk(sk)->mc_loop; 638 #if IS_ENABLED(CONFIG_IPV6) 639 case AF_INET6: 640 return inet6_sk(sk)->mc_loop; 641 #endif 642 } 643 WARN_ON(1); 644 return true; 645 } 646 EXPORT_SYMBOL(sk_mc_loop); 647 648 /* 649 * This is meant for all protocols to use and covers goings on 650 * at the socket level. Everything here is generic. 651 */ 652 653 int sock_setsockopt(struct socket *sock, int level, int optname, 654 char __user *optval, unsigned int optlen) 655 { 656 struct sock_txtime sk_txtime; 657 struct sock *sk = sock->sk; 658 int val; 659 int valbool; 660 struct linger ling; 661 int ret = 0; 662 663 /* 664 * Options without arguments 665 */ 666 667 if (optname == SO_BINDTODEVICE) 668 return sock_setbindtodevice(sk, optval, optlen); 669 670 if (optlen < sizeof(int)) 671 return -EINVAL; 672 673 if (get_user(val, (int __user *)optval)) 674 return -EFAULT; 675 676 valbool = val ? 1 : 0; 677 678 lock_sock(sk); 679 680 switch (optname) { 681 case SO_DEBUG: 682 if (val && !capable(CAP_NET_ADMIN)) 683 ret = -EACCES; 684 else 685 sock_valbool_flag(sk, SOCK_DBG, valbool); 686 break; 687 case SO_REUSEADDR: 688 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 689 break; 690 case SO_REUSEPORT: 691 sk->sk_reuseport = valbool; 692 break; 693 case SO_TYPE: 694 case SO_PROTOCOL: 695 case SO_DOMAIN: 696 case SO_ERROR: 697 ret = -ENOPROTOOPT; 698 break; 699 case SO_DONTROUTE: 700 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 701 break; 702 case SO_BROADCAST: 703 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 704 break; 705 case SO_SNDBUF: 706 /* Don't error on this BSD doesn't and if you think 707 * about it this is right. Otherwise apps have to 708 * play 'guess the biggest size' games. RCVBUF/SNDBUF 709 * are treated in BSD as hints 710 */ 711 val = min_t(u32, val, sysctl_wmem_max); 712 set_sndbuf: 713 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 714 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); 715 /* Wake up sending tasks if we upped the value. */ 716 sk->sk_write_space(sk); 717 break; 718 719 case SO_SNDBUFFORCE: 720 if (!capable(CAP_NET_ADMIN)) { 721 ret = -EPERM; 722 break; 723 } 724 goto set_sndbuf; 725 726 case SO_RCVBUF: 727 /* Don't error on this BSD doesn't and if you think 728 * about it this is right. Otherwise apps have to 729 * play 'guess the biggest size' games. RCVBUF/SNDBUF 730 * are treated in BSD as hints 731 */ 732 val = min_t(u32, val, sysctl_rmem_max); 733 set_rcvbuf: 734 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 735 /* 736 * We double it on the way in to account for 737 * "struct sk_buff" etc. overhead. Applications 738 * assume that the SO_RCVBUF setting they make will 739 * allow that much actual data to be received on that 740 * socket. 741 * 742 * Applications are unaware that "struct sk_buff" and 743 * other overheads allocate from the receive buffer 744 * during socket buffer allocation. 745 * 746 * And after considering the possible alternatives, 747 * returning the value we actually used in getsockopt 748 * is the most desirable behavior. 749 */ 750 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF); 751 break; 752 753 case SO_RCVBUFFORCE: 754 if (!capable(CAP_NET_ADMIN)) { 755 ret = -EPERM; 756 break; 757 } 758 goto set_rcvbuf; 759 760 case SO_KEEPALIVE: 761 if (sk->sk_prot->keepalive) 762 sk->sk_prot->keepalive(sk, valbool); 763 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 764 break; 765 766 case SO_OOBINLINE: 767 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 768 break; 769 770 case SO_NO_CHECK: 771 sk->sk_no_check_tx = valbool; 772 break; 773 774 case SO_PRIORITY: 775 if ((val >= 0 && val <= 6) || 776 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 777 sk->sk_priority = val; 778 else 779 ret = -EPERM; 780 break; 781 782 case SO_LINGER: 783 if (optlen < sizeof(ling)) { 784 ret = -EINVAL; /* 1003.1g */ 785 break; 786 } 787 if (copy_from_user(&ling, optval, sizeof(ling))) { 788 ret = -EFAULT; 789 break; 790 } 791 if (!ling.l_onoff) 792 sock_reset_flag(sk, SOCK_LINGER); 793 else { 794 #if (BITS_PER_LONG == 32) 795 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ) 796 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT; 797 else 798 #endif 799 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ; 800 sock_set_flag(sk, SOCK_LINGER); 801 } 802 break; 803 804 case SO_BSDCOMPAT: 805 sock_warn_obsolete_bsdism("setsockopt"); 806 break; 807 808 case SO_PASSCRED: 809 if (valbool) 810 set_bit(SOCK_PASSCRED, &sock->flags); 811 else 812 clear_bit(SOCK_PASSCRED, &sock->flags); 813 break; 814 815 case SO_TIMESTAMP: 816 case SO_TIMESTAMPNS: 817 if (valbool) { 818 if (optname == SO_TIMESTAMP) 819 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 820 else 821 sock_set_flag(sk, SOCK_RCVTSTAMPNS); 822 sock_set_flag(sk, SOCK_RCVTSTAMP); 823 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 824 } else { 825 sock_reset_flag(sk, SOCK_RCVTSTAMP); 826 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 827 } 828 break; 829 830 case SO_TIMESTAMPING: 831 if (val & ~SOF_TIMESTAMPING_MASK) { 832 ret = -EINVAL; 833 break; 834 } 835 836 if (val & SOF_TIMESTAMPING_OPT_ID && 837 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 838 if (sk->sk_protocol == IPPROTO_TCP && 839 sk->sk_type == SOCK_STREAM) { 840 if ((1 << sk->sk_state) & 841 (TCPF_CLOSE | TCPF_LISTEN)) { 842 ret = -EINVAL; 843 break; 844 } 845 sk->sk_tskey = tcp_sk(sk)->snd_una; 846 } else { 847 sk->sk_tskey = 0; 848 } 849 } 850 851 if (val & SOF_TIMESTAMPING_OPT_STATS && 852 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) { 853 ret = -EINVAL; 854 break; 855 } 856 857 sk->sk_tsflags = val; 858 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 859 sock_enable_timestamp(sk, 860 SOCK_TIMESTAMPING_RX_SOFTWARE); 861 else 862 sock_disable_timestamp(sk, 863 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 864 break; 865 866 case SO_RCVLOWAT: 867 if (val < 0) 868 val = INT_MAX; 869 if (sock->ops->set_rcvlowat) 870 ret = sock->ops->set_rcvlowat(sk, val); 871 else 872 sk->sk_rcvlowat = val ? : 1; 873 break; 874 875 case SO_RCVTIMEO: 876 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen); 877 break; 878 879 case SO_SNDTIMEO: 880 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen); 881 break; 882 883 case SO_ATTACH_FILTER: 884 ret = -EINVAL; 885 if (optlen == sizeof(struct sock_fprog)) { 886 struct sock_fprog fprog; 887 888 ret = -EFAULT; 889 if (copy_from_user(&fprog, optval, sizeof(fprog))) 890 break; 891 892 ret = sk_attach_filter(&fprog, sk); 893 } 894 break; 895 896 case SO_ATTACH_BPF: 897 ret = -EINVAL; 898 if (optlen == sizeof(u32)) { 899 u32 ufd; 900 901 ret = -EFAULT; 902 if (copy_from_user(&ufd, optval, sizeof(ufd))) 903 break; 904 905 ret = sk_attach_bpf(ufd, sk); 906 } 907 break; 908 909 case SO_ATTACH_REUSEPORT_CBPF: 910 ret = -EINVAL; 911 if (optlen == sizeof(struct sock_fprog)) { 912 struct sock_fprog fprog; 913 914 ret = -EFAULT; 915 if (copy_from_user(&fprog, optval, sizeof(fprog))) 916 break; 917 918 ret = sk_reuseport_attach_filter(&fprog, sk); 919 } 920 break; 921 922 case SO_ATTACH_REUSEPORT_EBPF: 923 ret = -EINVAL; 924 if (optlen == sizeof(u32)) { 925 u32 ufd; 926 927 ret = -EFAULT; 928 if (copy_from_user(&ufd, optval, sizeof(ufd))) 929 break; 930 931 ret = sk_reuseport_attach_bpf(ufd, sk); 932 } 933 break; 934 935 case SO_DETACH_FILTER: 936 ret = sk_detach_filter(sk); 937 break; 938 939 case SO_LOCK_FILTER: 940 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 941 ret = -EPERM; 942 else 943 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 944 break; 945 946 case SO_PASSSEC: 947 if (valbool) 948 set_bit(SOCK_PASSSEC, &sock->flags); 949 else 950 clear_bit(SOCK_PASSSEC, &sock->flags); 951 break; 952 case SO_MARK: 953 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 954 ret = -EPERM; 955 else 956 sk->sk_mark = val; 957 break; 958 959 case SO_RXQ_OVFL: 960 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 961 break; 962 963 case SO_WIFI_STATUS: 964 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 965 break; 966 967 case SO_PEEK_OFF: 968 if (sock->ops->set_peek_off) 969 ret = sock->ops->set_peek_off(sk, val); 970 else 971 ret = -EOPNOTSUPP; 972 break; 973 974 case SO_NOFCS: 975 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 976 break; 977 978 case SO_SELECT_ERR_QUEUE: 979 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 980 break; 981 982 #ifdef CONFIG_NET_RX_BUSY_POLL 983 case SO_BUSY_POLL: 984 /* allow unprivileged users to decrease the value */ 985 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN)) 986 ret = -EPERM; 987 else { 988 if (val < 0) 989 ret = -EINVAL; 990 else 991 sk->sk_ll_usec = val; 992 } 993 break; 994 #endif 995 996 case SO_MAX_PACING_RATE: 997 if (val != ~0U) 998 cmpxchg(&sk->sk_pacing_status, 999 SK_PACING_NONE, 1000 SK_PACING_NEEDED); 1001 sk->sk_max_pacing_rate = val; 1002 sk->sk_pacing_rate = min(sk->sk_pacing_rate, 1003 sk->sk_max_pacing_rate); 1004 break; 1005 1006 case SO_INCOMING_CPU: 1007 sk->sk_incoming_cpu = val; 1008 break; 1009 1010 case SO_CNX_ADVICE: 1011 if (val == 1) 1012 dst_negative_advice(sk); 1013 break; 1014 1015 case SO_ZEROCOPY: 1016 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1017 if (sk->sk_protocol != IPPROTO_TCP) 1018 ret = -ENOTSUPP; 1019 } else if (sk->sk_family != PF_RDS) { 1020 ret = -ENOTSUPP; 1021 } 1022 if (!ret) { 1023 if (val < 0 || val > 1) 1024 ret = -EINVAL; 1025 else 1026 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1027 } 1028 break; 1029 1030 case SO_TXTIME: 1031 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1032 ret = -EPERM; 1033 } else if (optlen != sizeof(struct sock_txtime)) { 1034 ret = -EINVAL; 1035 } else if (copy_from_user(&sk_txtime, optval, 1036 sizeof(struct sock_txtime))) { 1037 ret = -EFAULT; 1038 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1039 ret = -EINVAL; 1040 } else { 1041 sock_valbool_flag(sk, SOCK_TXTIME, true); 1042 sk->sk_clockid = sk_txtime.clockid; 1043 sk->sk_txtime_deadline_mode = 1044 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1045 sk->sk_txtime_report_errors = 1046 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1047 } 1048 break; 1049 1050 default: 1051 ret = -ENOPROTOOPT; 1052 break; 1053 } 1054 release_sock(sk); 1055 return ret; 1056 } 1057 EXPORT_SYMBOL(sock_setsockopt); 1058 1059 1060 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1061 struct ucred *ucred) 1062 { 1063 ucred->pid = pid_vnr(pid); 1064 ucred->uid = ucred->gid = -1; 1065 if (cred) { 1066 struct user_namespace *current_ns = current_user_ns(); 1067 1068 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1069 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1070 } 1071 } 1072 1073 static int groups_to_user(gid_t __user *dst, const struct group_info *src) 1074 { 1075 struct user_namespace *user_ns = current_user_ns(); 1076 int i; 1077 1078 for (i = 0; i < src->ngroups; i++) 1079 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i)) 1080 return -EFAULT; 1081 1082 return 0; 1083 } 1084 1085 int sock_getsockopt(struct socket *sock, int level, int optname, 1086 char __user *optval, int __user *optlen) 1087 { 1088 struct sock *sk = sock->sk; 1089 1090 union { 1091 int val; 1092 u64 val64; 1093 struct linger ling; 1094 struct timeval tm; 1095 struct sock_txtime txtime; 1096 } v; 1097 1098 int lv = sizeof(int); 1099 int len; 1100 1101 if (get_user(len, optlen)) 1102 return -EFAULT; 1103 if (len < 0) 1104 return -EINVAL; 1105 1106 memset(&v, 0, sizeof(v)); 1107 1108 switch (optname) { 1109 case SO_DEBUG: 1110 v.val = sock_flag(sk, SOCK_DBG); 1111 break; 1112 1113 case SO_DONTROUTE: 1114 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1115 break; 1116 1117 case SO_BROADCAST: 1118 v.val = sock_flag(sk, SOCK_BROADCAST); 1119 break; 1120 1121 case SO_SNDBUF: 1122 v.val = sk->sk_sndbuf; 1123 break; 1124 1125 case SO_RCVBUF: 1126 v.val = sk->sk_rcvbuf; 1127 break; 1128 1129 case SO_REUSEADDR: 1130 v.val = sk->sk_reuse; 1131 break; 1132 1133 case SO_REUSEPORT: 1134 v.val = sk->sk_reuseport; 1135 break; 1136 1137 case SO_KEEPALIVE: 1138 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1139 break; 1140 1141 case SO_TYPE: 1142 v.val = sk->sk_type; 1143 break; 1144 1145 case SO_PROTOCOL: 1146 v.val = sk->sk_protocol; 1147 break; 1148 1149 case SO_DOMAIN: 1150 v.val = sk->sk_family; 1151 break; 1152 1153 case SO_ERROR: 1154 v.val = -sock_error(sk); 1155 if (v.val == 0) 1156 v.val = xchg(&sk->sk_err_soft, 0); 1157 break; 1158 1159 case SO_OOBINLINE: 1160 v.val = sock_flag(sk, SOCK_URGINLINE); 1161 break; 1162 1163 case SO_NO_CHECK: 1164 v.val = sk->sk_no_check_tx; 1165 break; 1166 1167 case SO_PRIORITY: 1168 v.val = sk->sk_priority; 1169 break; 1170 1171 case SO_LINGER: 1172 lv = sizeof(v.ling); 1173 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1174 v.ling.l_linger = sk->sk_lingertime / HZ; 1175 break; 1176 1177 case SO_BSDCOMPAT: 1178 sock_warn_obsolete_bsdism("getsockopt"); 1179 break; 1180 1181 case SO_TIMESTAMP: 1182 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1183 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1184 break; 1185 1186 case SO_TIMESTAMPNS: 1187 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS); 1188 break; 1189 1190 case SO_TIMESTAMPING: 1191 v.val = sk->sk_tsflags; 1192 break; 1193 1194 case SO_RCVTIMEO: 1195 lv = sizeof(struct timeval); 1196 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) { 1197 v.tm.tv_sec = 0; 1198 v.tm.tv_usec = 0; 1199 } else { 1200 v.tm.tv_sec = sk->sk_rcvtimeo / HZ; 1201 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ; 1202 } 1203 break; 1204 1205 case SO_SNDTIMEO: 1206 lv = sizeof(struct timeval); 1207 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) { 1208 v.tm.tv_sec = 0; 1209 v.tm.tv_usec = 0; 1210 } else { 1211 v.tm.tv_sec = sk->sk_sndtimeo / HZ; 1212 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ; 1213 } 1214 break; 1215 1216 case SO_RCVLOWAT: 1217 v.val = sk->sk_rcvlowat; 1218 break; 1219 1220 case SO_SNDLOWAT: 1221 v.val = 1; 1222 break; 1223 1224 case SO_PASSCRED: 1225 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1226 break; 1227 1228 case SO_PEERCRED: 1229 { 1230 struct ucred peercred; 1231 if (len > sizeof(peercred)) 1232 len = sizeof(peercred); 1233 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1234 if (copy_to_user(optval, &peercred, len)) 1235 return -EFAULT; 1236 goto lenout; 1237 } 1238 1239 case SO_PEERGROUPS: 1240 { 1241 int ret, n; 1242 1243 if (!sk->sk_peer_cred) 1244 return -ENODATA; 1245 1246 n = sk->sk_peer_cred->group_info->ngroups; 1247 if (len < n * sizeof(gid_t)) { 1248 len = n * sizeof(gid_t); 1249 return put_user(len, optlen) ? -EFAULT : -ERANGE; 1250 } 1251 len = n * sizeof(gid_t); 1252 1253 ret = groups_to_user((gid_t __user *)optval, 1254 sk->sk_peer_cred->group_info); 1255 if (ret) 1256 return ret; 1257 goto lenout; 1258 } 1259 1260 case SO_PEERNAME: 1261 { 1262 char address[128]; 1263 1264 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2); 1265 if (lv < 0) 1266 return -ENOTCONN; 1267 if (lv < len) 1268 return -EINVAL; 1269 if (copy_to_user(optval, address, len)) 1270 return -EFAULT; 1271 goto lenout; 1272 } 1273 1274 /* Dubious BSD thing... Probably nobody even uses it, but 1275 * the UNIX standard wants it for whatever reason... -DaveM 1276 */ 1277 case SO_ACCEPTCONN: 1278 v.val = sk->sk_state == TCP_LISTEN; 1279 break; 1280 1281 case SO_PASSSEC: 1282 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1283 break; 1284 1285 case SO_PEERSEC: 1286 return security_socket_getpeersec_stream(sock, optval, optlen, len); 1287 1288 case SO_MARK: 1289 v.val = sk->sk_mark; 1290 break; 1291 1292 case SO_RXQ_OVFL: 1293 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1294 break; 1295 1296 case SO_WIFI_STATUS: 1297 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1298 break; 1299 1300 case SO_PEEK_OFF: 1301 if (!sock->ops->set_peek_off) 1302 return -EOPNOTSUPP; 1303 1304 v.val = sk->sk_peek_off; 1305 break; 1306 case SO_NOFCS: 1307 v.val = sock_flag(sk, SOCK_NOFCS); 1308 break; 1309 1310 case SO_BINDTODEVICE: 1311 return sock_getbindtodevice(sk, optval, optlen, len); 1312 1313 case SO_GET_FILTER: 1314 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len); 1315 if (len < 0) 1316 return len; 1317 1318 goto lenout; 1319 1320 case SO_LOCK_FILTER: 1321 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1322 break; 1323 1324 case SO_BPF_EXTENSIONS: 1325 v.val = bpf_tell_extensions(); 1326 break; 1327 1328 case SO_SELECT_ERR_QUEUE: 1329 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1330 break; 1331 1332 #ifdef CONFIG_NET_RX_BUSY_POLL 1333 case SO_BUSY_POLL: 1334 v.val = sk->sk_ll_usec; 1335 break; 1336 #endif 1337 1338 case SO_MAX_PACING_RATE: 1339 v.val = sk->sk_max_pacing_rate; 1340 break; 1341 1342 case SO_INCOMING_CPU: 1343 v.val = sk->sk_incoming_cpu; 1344 break; 1345 1346 case SO_MEMINFO: 1347 { 1348 u32 meminfo[SK_MEMINFO_VARS]; 1349 1350 if (get_user(len, optlen)) 1351 return -EFAULT; 1352 1353 sk_get_meminfo(sk, meminfo); 1354 1355 len = min_t(unsigned int, len, sizeof(meminfo)); 1356 if (copy_to_user(optval, &meminfo, len)) 1357 return -EFAULT; 1358 1359 goto lenout; 1360 } 1361 1362 #ifdef CONFIG_NET_RX_BUSY_POLL 1363 case SO_INCOMING_NAPI_ID: 1364 v.val = READ_ONCE(sk->sk_napi_id); 1365 1366 /* aggregate non-NAPI IDs down to 0 */ 1367 if (v.val < MIN_NAPI_ID) 1368 v.val = 0; 1369 1370 break; 1371 #endif 1372 1373 case SO_COOKIE: 1374 lv = sizeof(u64); 1375 if (len < lv) 1376 return -EINVAL; 1377 v.val64 = sock_gen_cookie(sk); 1378 break; 1379 1380 case SO_ZEROCOPY: 1381 v.val = sock_flag(sk, SOCK_ZEROCOPY); 1382 break; 1383 1384 case SO_TXTIME: 1385 lv = sizeof(v.txtime); 1386 v.txtime.clockid = sk->sk_clockid; 1387 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 1388 SOF_TXTIME_DEADLINE_MODE : 0; 1389 v.txtime.flags |= sk->sk_txtime_report_errors ? 1390 SOF_TXTIME_REPORT_ERRORS : 0; 1391 break; 1392 1393 default: 1394 /* We implement the SO_SNDLOWAT etc to not be settable 1395 * (1003.1g 7). 1396 */ 1397 return -ENOPROTOOPT; 1398 } 1399 1400 if (len > lv) 1401 len = lv; 1402 if (copy_to_user(optval, &v, len)) 1403 return -EFAULT; 1404 lenout: 1405 if (put_user(len, optlen)) 1406 return -EFAULT; 1407 return 0; 1408 } 1409 1410 /* 1411 * Initialize an sk_lock. 1412 * 1413 * (We also register the sk_lock with the lock validator.) 1414 */ 1415 static inline void sock_lock_init(struct sock *sk) 1416 { 1417 if (sk->sk_kern_sock) 1418 sock_lock_init_class_and_name( 1419 sk, 1420 af_family_kern_slock_key_strings[sk->sk_family], 1421 af_family_kern_slock_keys + sk->sk_family, 1422 af_family_kern_key_strings[sk->sk_family], 1423 af_family_kern_keys + sk->sk_family); 1424 else 1425 sock_lock_init_class_and_name( 1426 sk, 1427 af_family_slock_key_strings[sk->sk_family], 1428 af_family_slock_keys + sk->sk_family, 1429 af_family_key_strings[sk->sk_family], 1430 af_family_keys + sk->sk_family); 1431 } 1432 1433 /* 1434 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 1435 * even temporarly, because of RCU lookups. sk_node should also be left as is. 1436 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 1437 */ 1438 static void sock_copy(struct sock *nsk, const struct sock *osk) 1439 { 1440 #ifdef CONFIG_SECURITY_NETWORK 1441 void *sptr = nsk->sk_security; 1442 #endif 1443 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 1444 1445 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 1446 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 1447 1448 #ifdef CONFIG_SECURITY_NETWORK 1449 nsk->sk_security = sptr; 1450 security_sk_clone(osk, nsk); 1451 #endif 1452 } 1453 1454 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 1455 int family) 1456 { 1457 struct sock *sk; 1458 struct kmem_cache *slab; 1459 1460 slab = prot->slab; 1461 if (slab != NULL) { 1462 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 1463 if (!sk) 1464 return sk; 1465 if (priority & __GFP_ZERO) 1466 sk_prot_clear_nulls(sk, prot->obj_size); 1467 } else 1468 sk = kmalloc(prot->obj_size, priority); 1469 1470 if (sk != NULL) { 1471 if (security_sk_alloc(sk, family, priority)) 1472 goto out_free; 1473 1474 if (!try_module_get(prot->owner)) 1475 goto out_free_sec; 1476 sk_tx_queue_clear(sk); 1477 } 1478 1479 return sk; 1480 1481 out_free_sec: 1482 security_sk_free(sk); 1483 out_free: 1484 if (slab != NULL) 1485 kmem_cache_free(slab, sk); 1486 else 1487 kfree(sk); 1488 return NULL; 1489 } 1490 1491 static void sk_prot_free(struct proto *prot, struct sock *sk) 1492 { 1493 struct kmem_cache *slab; 1494 struct module *owner; 1495 1496 owner = prot->owner; 1497 slab = prot->slab; 1498 1499 cgroup_sk_free(&sk->sk_cgrp_data); 1500 mem_cgroup_sk_free(sk); 1501 security_sk_free(sk); 1502 if (slab != NULL) 1503 kmem_cache_free(slab, sk); 1504 else 1505 kfree(sk); 1506 module_put(owner); 1507 } 1508 1509 /** 1510 * sk_alloc - All socket objects are allocated here 1511 * @net: the applicable net namespace 1512 * @family: protocol family 1513 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1514 * @prot: struct proto associated with this new sock instance 1515 * @kern: is this to be a kernel socket? 1516 */ 1517 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 1518 struct proto *prot, int kern) 1519 { 1520 struct sock *sk; 1521 1522 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 1523 if (sk) { 1524 sk->sk_family = family; 1525 /* 1526 * See comment in struct sock definition to understand 1527 * why we need sk_prot_creator -acme 1528 */ 1529 sk->sk_prot = sk->sk_prot_creator = prot; 1530 sk->sk_kern_sock = kern; 1531 sock_lock_init(sk); 1532 sk->sk_net_refcnt = kern ? 0 : 1; 1533 if (likely(sk->sk_net_refcnt)) { 1534 get_net(net); 1535 sock_inuse_add(net, 1); 1536 } 1537 1538 sock_net_set(sk, net); 1539 refcount_set(&sk->sk_wmem_alloc, 1); 1540 1541 mem_cgroup_sk_alloc(sk); 1542 cgroup_sk_alloc(&sk->sk_cgrp_data); 1543 sock_update_classid(&sk->sk_cgrp_data); 1544 sock_update_netprioidx(&sk->sk_cgrp_data); 1545 } 1546 1547 return sk; 1548 } 1549 EXPORT_SYMBOL(sk_alloc); 1550 1551 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 1552 * grace period. This is the case for UDP sockets and TCP listeners. 1553 */ 1554 static void __sk_destruct(struct rcu_head *head) 1555 { 1556 struct sock *sk = container_of(head, struct sock, sk_rcu); 1557 struct sk_filter *filter; 1558 1559 if (sk->sk_destruct) 1560 sk->sk_destruct(sk); 1561 1562 filter = rcu_dereference_check(sk->sk_filter, 1563 refcount_read(&sk->sk_wmem_alloc) == 0); 1564 if (filter) { 1565 sk_filter_uncharge(sk, filter); 1566 RCU_INIT_POINTER(sk->sk_filter, NULL); 1567 } 1568 if (rcu_access_pointer(sk->sk_reuseport_cb)) 1569 reuseport_detach_sock(sk); 1570 1571 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 1572 1573 if (atomic_read(&sk->sk_omem_alloc)) 1574 pr_debug("%s: optmem leakage (%d bytes) detected\n", 1575 __func__, atomic_read(&sk->sk_omem_alloc)); 1576 1577 if (sk->sk_frag.page) { 1578 put_page(sk->sk_frag.page); 1579 sk->sk_frag.page = NULL; 1580 } 1581 1582 if (sk->sk_peer_cred) 1583 put_cred(sk->sk_peer_cred); 1584 put_pid(sk->sk_peer_pid); 1585 if (likely(sk->sk_net_refcnt)) 1586 put_net(sock_net(sk)); 1587 sk_prot_free(sk->sk_prot_creator, sk); 1588 } 1589 1590 void sk_destruct(struct sock *sk) 1591 { 1592 if (sock_flag(sk, SOCK_RCU_FREE)) 1593 call_rcu(&sk->sk_rcu, __sk_destruct); 1594 else 1595 __sk_destruct(&sk->sk_rcu); 1596 } 1597 1598 static void __sk_free(struct sock *sk) 1599 { 1600 if (likely(sk->sk_net_refcnt)) 1601 sock_inuse_add(sock_net(sk), -1); 1602 1603 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 1604 sock_diag_broadcast_destroy(sk); 1605 else 1606 sk_destruct(sk); 1607 } 1608 1609 void sk_free(struct sock *sk) 1610 { 1611 /* 1612 * We subtract one from sk_wmem_alloc and can know if 1613 * some packets are still in some tx queue. 1614 * If not null, sock_wfree() will call __sk_free(sk) later 1615 */ 1616 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 1617 __sk_free(sk); 1618 } 1619 EXPORT_SYMBOL(sk_free); 1620 1621 static void sk_init_common(struct sock *sk) 1622 { 1623 skb_queue_head_init(&sk->sk_receive_queue); 1624 skb_queue_head_init(&sk->sk_write_queue); 1625 skb_queue_head_init(&sk->sk_error_queue); 1626 1627 rwlock_init(&sk->sk_callback_lock); 1628 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 1629 af_rlock_keys + sk->sk_family, 1630 af_family_rlock_key_strings[sk->sk_family]); 1631 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 1632 af_wlock_keys + sk->sk_family, 1633 af_family_wlock_key_strings[sk->sk_family]); 1634 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 1635 af_elock_keys + sk->sk_family, 1636 af_family_elock_key_strings[sk->sk_family]); 1637 lockdep_set_class_and_name(&sk->sk_callback_lock, 1638 af_callback_keys + sk->sk_family, 1639 af_family_clock_key_strings[sk->sk_family]); 1640 } 1641 1642 /** 1643 * sk_clone_lock - clone a socket, and lock its clone 1644 * @sk: the socket to clone 1645 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1646 * 1647 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1648 */ 1649 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 1650 { 1651 struct sock *newsk; 1652 bool is_charged = true; 1653 1654 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family); 1655 if (newsk != NULL) { 1656 struct sk_filter *filter; 1657 1658 sock_copy(newsk, sk); 1659 1660 newsk->sk_prot_creator = sk->sk_prot; 1661 1662 /* SANITY */ 1663 if (likely(newsk->sk_net_refcnt)) 1664 get_net(sock_net(newsk)); 1665 sk_node_init(&newsk->sk_node); 1666 sock_lock_init(newsk); 1667 bh_lock_sock(newsk); 1668 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 1669 newsk->sk_backlog.len = 0; 1670 1671 atomic_set(&newsk->sk_rmem_alloc, 0); 1672 /* 1673 * sk_wmem_alloc set to one (see sk_free() and sock_wfree()) 1674 */ 1675 refcount_set(&newsk->sk_wmem_alloc, 1); 1676 atomic_set(&newsk->sk_omem_alloc, 0); 1677 sk_init_common(newsk); 1678 1679 newsk->sk_dst_cache = NULL; 1680 newsk->sk_dst_pending_confirm = 0; 1681 newsk->sk_wmem_queued = 0; 1682 newsk->sk_forward_alloc = 0; 1683 atomic_set(&newsk->sk_drops, 0); 1684 newsk->sk_send_head = NULL; 1685 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 1686 atomic_set(&newsk->sk_zckey, 0); 1687 1688 sock_reset_flag(newsk, SOCK_DONE); 1689 mem_cgroup_sk_alloc(newsk); 1690 cgroup_sk_alloc(&newsk->sk_cgrp_data); 1691 1692 rcu_read_lock(); 1693 filter = rcu_dereference(sk->sk_filter); 1694 if (filter != NULL) 1695 /* though it's an empty new sock, the charging may fail 1696 * if sysctl_optmem_max was changed between creation of 1697 * original socket and cloning 1698 */ 1699 is_charged = sk_filter_charge(newsk, filter); 1700 RCU_INIT_POINTER(newsk->sk_filter, filter); 1701 rcu_read_unlock(); 1702 1703 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 1704 /* We need to make sure that we don't uncharge the new 1705 * socket if we couldn't charge it in the first place 1706 * as otherwise we uncharge the parent's filter. 1707 */ 1708 if (!is_charged) 1709 RCU_INIT_POINTER(newsk->sk_filter, NULL); 1710 sk_free_unlock_clone(newsk); 1711 newsk = NULL; 1712 goto out; 1713 } 1714 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 1715 1716 newsk->sk_err = 0; 1717 newsk->sk_err_soft = 0; 1718 newsk->sk_priority = 0; 1719 newsk->sk_incoming_cpu = raw_smp_processor_id(); 1720 atomic64_set(&newsk->sk_cookie, 0); 1721 if (likely(newsk->sk_net_refcnt)) 1722 sock_inuse_add(sock_net(newsk), 1); 1723 1724 /* 1725 * Before updating sk_refcnt, we must commit prior changes to memory 1726 * (Documentation/RCU/rculist_nulls.txt for details) 1727 */ 1728 smp_wmb(); 1729 refcount_set(&newsk->sk_refcnt, 2); 1730 1731 /* 1732 * Increment the counter in the same struct proto as the master 1733 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that 1734 * is the same as sk->sk_prot->socks, as this field was copied 1735 * with memcpy). 1736 * 1737 * This _changes_ the previous behaviour, where 1738 * tcp_create_openreq_child always was incrementing the 1739 * equivalent to tcp_prot->socks (inet_sock_nr), so this have 1740 * to be taken into account in all callers. -acme 1741 */ 1742 sk_refcnt_debug_inc(newsk); 1743 sk_set_socket(newsk, NULL); 1744 newsk->sk_wq = NULL; 1745 1746 if (newsk->sk_prot->sockets_allocated) 1747 sk_sockets_allocated_inc(newsk); 1748 1749 if (sock_needs_netstamp(sk) && 1750 newsk->sk_flags & SK_FLAGS_TIMESTAMP) 1751 net_enable_timestamp(); 1752 } 1753 out: 1754 return newsk; 1755 } 1756 EXPORT_SYMBOL_GPL(sk_clone_lock); 1757 1758 void sk_free_unlock_clone(struct sock *sk) 1759 { 1760 /* It is still raw copy of parent, so invalidate 1761 * destructor and make plain sk_free() */ 1762 sk->sk_destruct = NULL; 1763 bh_unlock_sock(sk); 1764 sk_free(sk); 1765 } 1766 EXPORT_SYMBOL_GPL(sk_free_unlock_clone); 1767 1768 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 1769 { 1770 u32 max_segs = 1; 1771 1772 sk_dst_set(sk, dst); 1773 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps; 1774 if (sk->sk_route_caps & NETIF_F_GSO) 1775 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 1776 sk->sk_route_caps &= ~sk->sk_route_nocaps; 1777 if (sk_can_gso(sk)) { 1778 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 1779 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 1780 } else { 1781 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 1782 sk->sk_gso_max_size = dst->dev->gso_max_size; 1783 max_segs = max_t(u32, dst->dev->gso_max_segs, 1); 1784 } 1785 } 1786 sk->sk_gso_max_segs = max_segs; 1787 } 1788 EXPORT_SYMBOL_GPL(sk_setup_caps); 1789 1790 /* 1791 * Simple resource managers for sockets. 1792 */ 1793 1794 1795 /* 1796 * Write buffer destructor automatically called from kfree_skb. 1797 */ 1798 void sock_wfree(struct sk_buff *skb) 1799 { 1800 struct sock *sk = skb->sk; 1801 unsigned int len = skb->truesize; 1802 1803 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 1804 /* 1805 * Keep a reference on sk_wmem_alloc, this will be released 1806 * after sk_write_space() call 1807 */ 1808 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 1809 sk->sk_write_space(sk); 1810 len = 1; 1811 } 1812 /* 1813 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 1814 * could not do because of in-flight packets 1815 */ 1816 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 1817 __sk_free(sk); 1818 } 1819 EXPORT_SYMBOL(sock_wfree); 1820 1821 /* This variant of sock_wfree() is used by TCP, 1822 * since it sets SOCK_USE_WRITE_QUEUE. 1823 */ 1824 void __sock_wfree(struct sk_buff *skb) 1825 { 1826 struct sock *sk = skb->sk; 1827 1828 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 1829 __sk_free(sk); 1830 } 1831 1832 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 1833 { 1834 skb_orphan(skb); 1835 skb->sk = sk; 1836 #ifdef CONFIG_INET 1837 if (unlikely(!sk_fullsock(sk))) { 1838 skb->destructor = sock_edemux; 1839 sock_hold(sk); 1840 return; 1841 } 1842 #endif 1843 skb->destructor = sock_wfree; 1844 skb_set_hash_from_sk(skb, sk); 1845 /* 1846 * We used to take a refcount on sk, but following operation 1847 * is enough to guarantee sk_free() wont free this sock until 1848 * all in-flight packets are completed 1849 */ 1850 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 1851 } 1852 EXPORT_SYMBOL(skb_set_owner_w); 1853 1854 /* This helper is used by netem, as it can hold packets in its 1855 * delay queue. We want to allow the owner socket to send more 1856 * packets, as if they were already TX completed by a typical driver. 1857 * But we also want to keep skb->sk set because some packet schedulers 1858 * rely on it (sch_fq for example). 1859 */ 1860 void skb_orphan_partial(struct sk_buff *skb) 1861 { 1862 if (skb_is_tcp_pure_ack(skb)) 1863 return; 1864 1865 if (skb->destructor == sock_wfree 1866 #ifdef CONFIG_INET 1867 || skb->destructor == tcp_wfree 1868 #endif 1869 ) { 1870 struct sock *sk = skb->sk; 1871 1872 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 1873 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)); 1874 skb->destructor = sock_efree; 1875 } 1876 } else { 1877 skb_orphan(skb); 1878 } 1879 } 1880 EXPORT_SYMBOL(skb_orphan_partial); 1881 1882 /* 1883 * Read buffer destructor automatically called from kfree_skb. 1884 */ 1885 void sock_rfree(struct sk_buff *skb) 1886 { 1887 struct sock *sk = skb->sk; 1888 unsigned int len = skb->truesize; 1889 1890 atomic_sub(len, &sk->sk_rmem_alloc); 1891 sk_mem_uncharge(sk, len); 1892 } 1893 EXPORT_SYMBOL(sock_rfree); 1894 1895 /* 1896 * Buffer destructor for skbs that are not used directly in read or write 1897 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 1898 */ 1899 void sock_efree(struct sk_buff *skb) 1900 { 1901 sock_put(skb->sk); 1902 } 1903 EXPORT_SYMBOL(sock_efree); 1904 1905 kuid_t sock_i_uid(struct sock *sk) 1906 { 1907 kuid_t uid; 1908 1909 read_lock_bh(&sk->sk_callback_lock); 1910 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 1911 read_unlock_bh(&sk->sk_callback_lock); 1912 return uid; 1913 } 1914 EXPORT_SYMBOL(sock_i_uid); 1915 1916 unsigned long sock_i_ino(struct sock *sk) 1917 { 1918 unsigned long ino; 1919 1920 read_lock_bh(&sk->sk_callback_lock); 1921 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 1922 read_unlock_bh(&sk->sk_callback_lock); 1923 return ino; 1924 } 1925 EXPORT_SYMBOL(sock_i_ino); 1926 1927 /* 1928 * Allocate a skb from the socket's send buffer. 1929 */ 1930 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 1931 gfp_t priority) 1932 { 1933 if (force || refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) { 1934 struct sk_buff *skb = alloc_skb(size, priority); 1935 if (skb) { 1936 skb_set_owner_w(skb, sk); 1937 return skb; 1938 } 1939 } 1940 return NULL; 1941 } 1942 EXPORT_SYMBOL(sock_wmalloc); 1943 1944 static void sock_ofree(struct sk_buff *skb) 1945 { 1946 struct sock *sk = skb->sk; 1947 1948 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 1949 } 1950 1951 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 1952 gfp_t priority) 1953 { 1954 struct sk_buff *skb; 1955 1956 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 1957 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 1958 sysctl_optmem_max) 1959 return NULL; 1960 1961 skb = alloc_skb(size, priority); 1962 if (!skb) 1963 return NULL; 1964 1965 atomic_add(skb->truesize, &sk->sk_omem_alloc); 1966 skb->sk = sk; 1967 skb->destructor = sock_ofree; 1968 return skb; 1969 } 1970 1971 /* 1972 * Allocate a memory block from the socket's option memory buffer. 1973 */ 1974 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 1975 { 1976 if ((unsigned int)size <= sysctl_optmem_max && 1977 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { 1978 void *mem; 1979 /* First do the add, to avoid the race if kmalloc 1980 * might sleep. 1981 */ 1982 atomic_add(size, &sk->sk_omem_alloc); 1983 mem = kmalloc(size, priority); 1984 if (mem) 1985 return mem; 1986 atomic_sub(size, &sk->sk_omem_alloc); 1987 } 1988 return NULL; 1989 } 1990 EXPORT_SYMBOL(sock_kmalloc); 1991 1992 /* Free an option memory block. Note, we actually want the inline 1993 * here as this allows gcc to detect the nullify and fold away the 1994 * condition entirely. 1995 */ 1996 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 1997 const bool nullify) 1998 { 1999 if (WARN_ON_ONCE(!mem)) 2000 return; 2001 if (nullify) 2002 kzfree(mem); 2003 else 2004 kfree(mem); 2005 atomic_sub(size, &sk->sk_omem_alloc); 2006 } 2007 2008 void sock_kfree_s(struct sock *sk, void *mem, int size) 2009 { 2010 __sock_kfree_s(sk, mem, size, false); 2011 } 2012 EXPORT_SYMBOL(sock_kfree_s); 2013 2014 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2015 { 2016 __sock_kfree_s(sk, mem, size, true); 2017 } 2018 EXPORT_SYMBOL(sock_kzfree_s); 2019 2020 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2021 I think, these locks should be removed for datagram sockets. 2022 */ 2023 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2024 { 2025 DEFINE_WAIT(wait); 2026 2027 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2028 for (;;) { 2029 if (!timeo) 2030 break; 2031 if (signal_pending(current)) 2032 break; 2033 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2034 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2035 if (refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) 2036 break; 2037 if (sk->sk_shutdown & SEND_SHUTDOWN) 2038 break; 2039 if (sk->sk_err) 2040 break; 2041 timeo = schedule_timeout(timeo); 2042 } 2043 finish_wait(sk_sleep(sk), &wait); 2044 return timeo; 2045 } 2046 2047 2048 /* 2049 * Generic send/receive buffer handlers 2050 */ 2051 2052 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2053 unsigned long data_len, int noblock, 2054 int *errcode, int max_page_order) 2055 { 2056 struct sk_buff *skb; 2057 long timeo; 2058 int err; 2059 2060 timeo = sock_sndtimeo(sk, noblock); 2061 for (;;) { 2062 err = sock_error(sk); 2063 if (err != 0) 2064 goto failure; 2065 2066 err = -EPIPE; 2067 if (sk->sk_shutdown & SEND_SHUTDOWN) 2068 goto failure; 2069 2070 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf) 2071 break; 2072 2073 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2074 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2075 err = -EAGAIN; 2076 if (!timeo) 2077 goto failure; 2078 if (signal_pending(current)) 2079 goto interrupted; 2080 timeo = sock_wait_for_wmem(sk, timeo); 2081 } 2082 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 2083 errcode, sk->sk_allocation); 2084 if (skb) 2085 skb_set_owner_w(skb, sk); 2086 return skb; 2087 2088 interrupted: 2089 err = sock_intr_errno(timeo); 2090 failure: 2091 *errcode = err; 2092 return NULL; 2093 } 2094 EXPORT_SYMBOL(sock_alloc_send_pskb); 2095 2096 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size, 2097 int noblock, int *errcode) 2098 { 2099 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0); 2100 } 2101 EXPORT_SYMBOL(sock_alloc_send_skb); 2102 2103 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, 2104 struct sockcm_cookie *sockc) 2105 { 2106 u32 tsflags; 2107 2108 switch (cmsg->cmsg_type) { 2109 case SO_MARK: 2110 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 2111 return -EPERM; 2112 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2113 return -EINVAL; 2114 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 2115 break; 2116 case SO_TIMESTAMPING: 2117 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2118 return -EINVAL; 2119 2120 tsflags = *(u32 *)CMSG_DATA(cmsg); 2121 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 2122 return -EINVAL; 2123 2124 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 2125 sockc->tsflags |= tsflags; 2126 break; 2127 case SCM_TXTIME: 2128 if (!sock_flag(sk, SOCK_TXTIME)) 2129 return -EINVAL; 2130 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 2131 return -EINVAL; 2132 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); 2133 break; 2134 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 2135 case SCM_RIGHTS: 2136 case SCM_CREDENTIALS: 2137 break; 2138 default: 2139 return -EINVAL; 2140 } 2141 return 0; 2142 } 2143 EXPORT_SYMBOL(__sock_cmsg_send); 2144 2145 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 2146 struct sockcm_cookie *sockc) 2147 { 2148 struct cmsghdr *cmsg; 2149 int ret; 2150 2151 for_each_cmsghdr(cmsg, msg) { 2152 if (!CMSG_OK(msg, cmsg)) 2153 return -EINVAL; 2154 if (cmsg->cmsg_level != SOL_SOCKET) 2155 continue; 2156 ret = __sock_cmsg_send(sk, msg, cmsg, sockc); 2157 if (ret) 2158 return ret; 2159 } 2160 return 0; 2161 } 2162 EXPORT_SYMBOL(sock_cmsg_send); 2163 2164 static void sk_enter_memory_pressure(struct sock *sk) 2165 { 2166 if (!sk->sk_prot->enter_memory_pressure) 2167 return; 2168 2169 sk->sk_prot->enter_memory_pressure(sk); 2170 } 2171 2172 static void sk_leave_memory_pressure(struct sock *sk) 2173 { 2174 if (sk->sk_prot->leave_memory_pressure) { 2175 sk->sk_prot->leave_memory_pressure(sk); 2176 } else { 2177 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 2178 2179 if (memory_pressure && *memory_pressure) 2180 *memory_pressure = 0; 2181 } 2182 } 2183 2184 /* On 32bit arches, an skb frag is limited to 2^15 */ 2185 #define SKB_FRAG_PAGE_ORDER get_order(32768) 2186 2187 /** 2188 * skb_page_frag_refill - check that a page_frag contains enough room 2189 * @sz: minimum size of the fragment we want to get 2190 * @pfrag: pointer to page_frag 2191 * @gfp: priority for memory allocation 2192 * 2193 * Note: While this allocator tries to use high order pages, there is 2194 * no guarantee that allocations succeed. Therefore, @sz MUST be 2195 * less or equal than PAGE_SIZE. 2196 */ 2197 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 2198 { 2199 if (pfrag->page) { 2200 if (page_ref_count(pfrag->page) == 1) { 2201 pfrag->offset = 0; 2202 return true; 2203 } 2204 if (pfrag->offset + sz <= pfrag->size) 2205 return true; 2206 put_page(pfrag->page); 2207 } 2208 2209 pfrag->offset = 0; 2210 if (SKB_FRAG_PAGE_ORDER) { 2211 /* Avoid direct reclaim but allow kswapd to wake */ 2212 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 2213 __GFP_COMP | __GFP_NOWARN | 2214 __GFP_NORETRY, 2215 SKB_FRAG_PAGE_ORDER); 2216 if (likely(pfrag->page)) { 2217 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 2218 return true; 2219 } 2220 } 2221 pfrag->page = alloc_page(gfp); 2222 if (likely(pfrag->page)) { 2223 pfrag->size = PAGE_SIZE; 2224 return true; 2225 } 2226 return false; 2227 } 2228 EXPORT_SYMBOL(skb_page_frag_refill); 2229 2230 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2231 { 2232 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2233 return true; 2234 2235 sk_enter_memory_pressure(sk); 2236 sk_stream_moderate_sndbuf(sk); 2237 return false; 2238 } 2239 EXPORT_SYMBOL(sk_page_frag_refill); 2240 2241 int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg, 2242 int sg_start, int *sg_curr_index, unsigned int *sg_curr_size, 2243 int first_coalesce) 2244 { 2245 int sg_curr = *sg_curr_index, use = 0, rc = 0; 2246 unsigned int size = *sg_curr_size; 2247 struct page_frag *pfrag; 2248 struct scatterlist *sge; 2249 2250 len -= size; 2251 pfrag = sk_page_frag(sk); 2252 2253 while (len > 0) { 2254 unsigned int orig_offset; 2255 2256 if (!sk_page_frag_refill(sk, pfrag)) { 2257 rc = -ENOMEM; 2258 goto out; 2259 } 2260 2261 use = min_t(int, len, pfrag->size - pfrag->offset); 2262 2263 if (!sk_wmem_schedule(sk, use)) { 2264 rc = -ENOMEM; 2265 goto out; 2266 } 2267 2268 sk_mem_charge(sk, use); 2269 size += use; 2270 orig_offset = pfrag->offset; 2271 pfrag->offset += use; 2272 2273 sge = sg + sg_curr - 1; 2274 if (sg_curr > first_coalesce && sg_page(sge) == pfrag->page && 2275 sge->offset + sge->length == orig_offset) { 2276 sge->length += use; 2277 } else { 2278 sge = sg + sg_curr; 2279 sg_unmark_end(sge); 2280 sg_set_page(sge, pfrag->page, use, orig_offset); 2281 get_page(pfrag->page); 2282 sg_curr++; 2283 2284 if (sg_curr == MAX_SKB_FRAGS) 2285 sg_curr = 0; 2286 2287 if (sg_curr == sg_start) { 2288 rc = -ENOSPC; 2289 break; 2290 } 2291 } 2292 2293 len -= use; 2294 } 2295 out: 2296 *sg_curr_size = size; 2297 *sg_curr_index = sg_curr; 2298 return rc; 2299 } 2300 EXPORT_SYMBOL(sk_alloc_sg); 2301 2302 static void __lock_sock(struct sock *sk) 2303 __releases(&sk->sk_lock.slock) 2304 __acquires(&sk->sk_lock.slock) 2305 { 2306 DEFINE_WAIT(wait); 2307 2308 for (;;) { 2309 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2310 TASK_UNINTERRUPTIBLE); 2311 spin_unlock_bh(&sk->sk_lock.slock); 2312 schedule(); 2313 spin_lock_bh(&sk->sk_lock.slock); 2314 if (!sock_owned_by_user(sk)) 2315 break; 2316 } 2317 finish_wait(&sk->sk_lock.wq, &wait); 2318 } 2319 2320 static void __release_sock(struct sock *sk) 2321 __releases(&sk->sk_lock.slock) 2322 __acquires(&sk->sk_lock.slock) 2323 { 2324 struct sk_buff *skb, *next; 2325 2326 while ((skb = sk->sk_backlog.head) != NULL) { 2327 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2328 2329 spin_unlock_bh(&sk->sk_lock.slock); 2330 2331 do { 2332 next = skb->next; 2333 prefetch(next); 2334 WARN_ON_ONCE(skb_dst_is_noref(skb)); 2335 skb->next = NULL; 2336 sk_backlog_rcv(sk, skb); 2337 2338 cond_resched(); 2339 2340 skb = next; 2341 } while (skb != NULL); 2342 2343 spin_lock_bh(&sk->sk_lock.slock); 2344 } 2345 2346 /* 2347 * Doing the zeroing here guarantee we can not loop forever 2348 * while a wild producer attempts to flood us. 2349 */ 2350 sk->sk_backlog.len = 0; 2351 } 2352 2353 void __sk_flush_backlog(struct sock *sk) 2354 { 2355 spin_lock_bh(&sk->sk_lock.slock); 2356 __release_sock(sk); 2357 spin_unlock_bh(&sk->sk_lock.slock); 2358 } 2359 2360 /** 2361 * sk_wait_data - wait for data to arrive at sk_receive_queue 2362 * @sk: sock to wait on 2363 * @timeo: for how long 2364 * @skb: last skb seen on sk_receive_queue 2365 * 2366 * Now socket state including sk->sk_err is changed only under lock, 2367 * hence we may omit checks after joining wait queue. 2368 * We check receive queue before schedule() only as optimization; 2369 * it is very likely that release_sock() added new data. 2370 */ 2371 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 2372 { 2373 DEFINE_WAIT_FUNC(wait, woken_wake_function); 2374 int rc; 2375 2376 add_wait_queue(sk_sleep(sk), &wait); 2377 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2378 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 2379 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2380 remove_wait_queue(sk_sleep(sk), &wait); 2381 return rc; 2382 } 2383 EXPORT_SYMBOL(sk_wait_data); 2384 2385 /** 2386 * __sk_mem_raise_allocated - increase memory_allocated 2387 * @sk: socket 2388 * @size: memory size to allocate 2389 * @amt: pages to allocate 2390 * @kind: allocation type 2391 * 2392 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc 2393 */ 2394 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 2395 { 2396 struct proto *prot = sk->sk_prot; 2397 long allocated = sk_memory_allocated_add(sk, amt); 2398 bool charged = true; 2399 2400 if (mem_cgroup_sockets_enabled && sk->sk_memcg && 2401 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt))) 2402 goto suppress_allocation; 2403 2404 /* Under limit. */ 2405 if (allocated <= sk_prot_mem_limits(sk, 0)) { 2406 sk_leave_memory_pressure(sk); 2407 return 1; 2408 } 2409 2410 /* Under pressure. */ 2411 if (allocated > sk_prot_mem_limits(sk, 1)) 2412 sk_enter_memory_pressure(sk); 2413 2414 /* Over hard limit. */ 2415 if (allocated > sk_prot_mem_limits(sk, 2)) 2416 goto suppress_allocation; 2417 2418 /* guarantee minimum buffer size under pressure */ 2419 if (kind == SK_MEM_RECV) { 2420 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 2421 return 1; 2422 2423 } else { /* SK_MEM_SEND */ 2424 int wmem0 = sk_get_wmem0(sk, prot); 2425 2426 if (sk->sk_type == SOCK_STREAM) { 2427 if (sk->sk_wmem_queued < wmem0) 2428 return 1; 2429 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 2430 return 1; 2431 } 2432 } 2433 2434 if (sk_has_memory_pressure(sk)) { 2435 int alloc; 2436 2437 if (!sk_under_memory_pressure(sk)) 2438 return 1; 2439 alloc = sk_sockets_allocated_read_positive(sk); 2440 if (sk_prot_mem_limits(sk, 2) > alloc * 2441 sk_mem_pages(sk->sk_wmem_queued + 2442 atomic_read(&sk->sk_rmem_alloc) + 2443 sk->sk_forward_alloc)) 2444 return 1; 2445 } 2446 2447 suppress_allocation: 2448 2449 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 2450 sk_stream_moderate_sndbuf(sk); 2451 2452 /* Fail only if socket is _under_ its sndbuf. 2453 * In this case we cannot block, so that we have to fail. 2454 */ 2455 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) 2456 return 1; 2457 } 2458 2459 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged)) 2460 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 2461 2462 sk_memory_allocated_sub(sk, amt); 2463 2464 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2465 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt); 2466 2467 return 0; 2468 } 2469 EXPORT_SYMBOL(__sk_mem_raise_allocated); 2470 2471 /** 2472 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 2473 * @sk: socket 2474 * @size: memory size to allocate 2475 * @kind: allocation type 2476 * 2477 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 2478 * rmem allocation. This function assumes that protocols which have 2479 * memory_pressure use sk_wmem_queued as write buffer accounting. 2480 */ 2481 int __sk_mem_schedule(struct sock *sk, int size, int kind) 2482 { 2483 int ret, amt = sk_mem_pages(size); 2484 2485 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT; 2486 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 2487 if (!ret) 2488 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT; 2489 return ret; 2490 } 2491 EXPORT_SYMBOL(__sk_mem_schedule); 2492 2493 /** 2494 * __sk_mem_reduce_allocated - reclaim memory_allocated 2495 * @sk: socket 2496 * @amount: number of quanta 2497 * 2498 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 2499 */ 2500 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 2501 { 2502 sk_memory_allocated_sub(sk, amount); 2503 2504 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2505 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 2506 2507 if (sk_under_memory_pressure(sk) && 2508 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 2509 sk_leave_memory_pressure(sk); 2510 } 2511 EXPORT_SYMBOL(__sk_mem_reduce_allocated); 2512 2513 /** 2514 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 2515 * @sk: socket 2516 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple) 2517 */ 2518 void __sk_mem_reclaim(struct sock *sk, int amount) 2519 { 2520 amount >>= SK_MEM_QUANTUM_SHIFT; 2521 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT; 2522 __sk_mem_reduce_allocated(sk, amount); 2523 } 2524 EXPORT_SYMBOL(__sk_mem_reclaim); 2525 2526 int sk_set_peek_off(struct sock *sk, int val) 2527 { 2528 sk->sk_peek_off = val; 2529 return 0; 2530 } 2531 EXPORT_SYMBOL_GPL(sk_set_peek_off); 2532 2533 /* 2534 * Set of default routines for initialising struct proto_ops when 2535 * the protocol does not support a particular function. In certain 2536 * cases where it makes no sense for a protocol to have a "do nothing" 2537 * function, some default processing is provided. 2538 */ 2539 2540 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 2541 { 2542 return -EOPNOTSUPP; 2543 } 2544 EXPORT_SYMBOL(sock_no_bind); 2545 2546 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 2547 int len, int flags) 2548 { 2549 return -EOPNOTSUPP; 2550 } 2551 EXPORT_SYMBOL(sock_no_connect); 2552 2553 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 2554 { 2555 return -EOPNOTSUPP; 2556 } 2557 EXPORT_SYMBOL(sock_no_socketpair); 2558 2559 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, 2560 bool kern) 2561 { 2562 return -EOPNOTSUPP; 2563 } 2564 EXPORT_SYMBOL(sock_no_accept); 2565 2566 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 2567 int peer) 2568 { 2569 return -EOPNOTSUPP; 2570 } 2571 EXPORT_SYMBOL(sock_no_getname); 2572 2573 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 2574 { 2575 return -EOPNOTSUPP; 2576 } 2577 EXPORT_SYMBOL(sock_no_ioctl); 2578 2579 int sock_no_listen(struct socket *sock, int backlog) 2580 { 2581 return -EOPNOTSUPP; 2582 } 2583 EXPORT_SYMBOL(sock_no_listen); 2584 2585 int sock_no_shutdown(struct socket *sock, int how) 2586 { 2587 return -EOPNOTSUPP; 2588 } 2589 EXPORT_SYMBOL(sock_no_shutdown); 2590 2591 int sock_no_setsockopt(struct socket *sock, int level, int optname, 2592 char __user *optval, unsigned int optlen) 2593 { 2594 return -EOPNOTSUPP; 2595 } 2596 EXPORT_SYMBOL(sock_no_setsockopt); 2597 2598 int sock_no_getsockopt(struct socket *sock, int level, int optname, 2599 char __user *optval, int __user *optlen) 2600 { 2601 return -EOPNOTSUPP; 2602 } 2603 EXPORT_SYMBOL(sock_no_getsockopt); 2604 2605 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 2606 { 2607 return -EOPNOTSUPP; 2608 } 2609 EXPORT_SYMBOL(sock_no_sendmsg); 2610 2611 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 2612 { 2613 return -EOPNOTSUPP; 2614 } 2615 EXPORT_SYMBOL(sock_no_sendmsg_locked); 2616 2617 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 2618 int flags) 2619 { 2620 return -EOPNOTSUPP; 2621 } 2622 EXPORT_SYMBOL(sock_no_recvmsg); 2623 2624 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 2625 { 2626 /* Mirror missing mmap method error code */ 2627 return -ENODEV; 2628 } 2629 EXPORT_SYMBOL(sock_no_mmap); 2630 2631 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) 2632 { 2633 ssize_t res; 2634 struct msghdr msg = {.msg_flags = flags}; 2635 struct kvec iov; 2636 char *kaddr = kmap(page); 2637 iov.iov_base = kaddr + offset; 2638 iov.iov_len = size; 2639 res = kernel_sendmsg(sock, &msg, &iov, 1, size); 2640 kunmap(page); 2641 return res; 2642 } 2643 EXPORT_SYMBOL(sock_no_sendpage); 2644 2645 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page, 2646 int offset, size_t size, int flags) 2647 { 2648 ssize_t res; 2649 struct msghdr msg = {.msg_flags = flags}; 2650 struct kvec iov; 2651 char *kaddr = kmap(page); 2652 2653 iov.iov_base = kaddr + offset; 2654 iov.iov_len = size; 2655 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size); 2656 kunmap(page); 2657 return res; 2658 } 2659 EXPORT_SYMBOL(sock_no_sendpage_locked); 2660 2661 /* 2662 * Default Socket Callbacks 2663 */ 2664 2665 static void sock_def_wakeup(struct sock *sk) 2666 { 2667 struct socket_wq *wq; 2668 2669 rcu_read_lock(); 2670 wq = rcu_dereference(sk->sk_wq); 2671 if (skwq_has_sleeper(wq)) 2672 wake_up_interruptible_all(&wq->wait); 2673 rcu_read_unlock(); 2674 } 2675 2676 static void sock_def_error_report(struct sock *sk) 2677 { 2678 struct socket_wq *wq; 2679 2680 rcu_read_lock(); 2681 wq = rcu_dereference(sk->sk_wq); 2682 if (skwq_has_sleeper(wq)) 2683 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 2684 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 2685 rcu_read_unlock(); 2686 } 2687 2688 static void sock_def_readable(struct sock *sk) 2689 { 2690 struct socket_wq *wq; 2691 2692 rcu_read_lock(); 2693 wq = rcu_dereference(sk->sk_wq); 2694 if (skwq_has_sleeper(wq)) 2695 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 2696 EPOLLRDNORM | EPOLLRDBAND); 2697 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 2698 rcu_read_unlock(); 2699 } 2700 2701 static void sock_def_write_space(struct sock *sk) 2702 { 2703 struct socket_wq *wq; 2704 2705 rcu_read_lock(); 2706 2707 /* Do not wake up a writer until he can make "significant" 2708 * progress. --DaveM 2709 */ 2710 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) { 2711 wq = rcu_dereference(sk->sk_wq); 2712 if (skwq_has_sleeper(wq)) 2713 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 2714 EPOLLWRNORM | EPOLLWRBAND); 2715 2716 /* Should agree with poll, otherwise some programs break */ 2717 if (sock_writeable(sk)) 2718 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 2719 } 2720 2721 rcu_read_unlock(); 2722 } 2723 2724 static void sock_def_destruct(struct sock *sk) 2725 { 2726 } 2727 2728 void sk_send_sigurg(struct sock *sk) 2729 { 2730 if (sk->sk_socket && sk->sk_socket->file) 2731 if (send_sigurg(&sk->sk_socket->file->f_owner)) 2732 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 2733 } 2734 EXPORT_SYMBOL(sk_send_sigurg); 2735 2736 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 2737 unsigned long expires) 2738 { 2739 if (!mod_timer(timer, expires)) 2740 sock_hold(sk); 2741 } 2742 EXPORT_SYMBOL(sk_reset_timer); 2743 2744 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 2745 { 2746 if (del_timer(timer)) 2747 __sock_put(sk); 2748 } 2749 EXPORT_SYMBOL(sk_stop_timer); 2750 2751 void sock_init_data(struct socket *sock, struct sock *sk) 2752 { 2753 sk_init_common(sk); 2754 sk->sk_send_head = NULL; 2755 2756 timer_setup(&sk->sk_timer, NULL, 0); 2757 2758 sk->sk_allocation = GFP_KERNEL; 2759 sk->sk_rcvbuf = sysctl_rmem_default; 2760 sk->sk_sndbuf = sysctl_wmem_default; 2761 sk->sk_state = TCP_CLOSE; 2762 sk_set_socket(sk, sock); 2763 2764 sock_set_flag(sk, SOCK_ZAPPED); 2765 2766 if (sock) { 2767 sk->sk_type = sock->type; 2768 sk->sk_wq = sock->wq; 2769 sock->sk = sk; 2770 sk->sk_uid = SOCK_INODE(sock)->i_uid; 2771 } else { 2772 sk->sk_wq = NULL; 2773 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0); 2774 } 2775 2776 rwlock_init(&sk->sk_callback_lock); 2777 if (sk->sk_kern_sock) 2778 lockdep_set_class_and_name( 2779 &sk->sk_callback_lock, 2780 af_kern_callback_keys + sk->sk_family, 2781 af_family_kern_clock_key_strings[sk->sk_family]); 2782 else 2783 lockdep_set_class_and_name( 2784 &sk->sk_callback_lock, 2785 af_callback_keys + sk->sk_family, 2786 af_family_clock_key_strings[sk->sk_family]); 2787 2788 sk->sk_state_change = sock_def_wakeup; 2789 sk->sk_data_ready = sock_def_readable; 2790 sk->sk_write_space = sock_def_write_space; 2791 sk->sk_error_report = sock_def_error_report; 2792 sk->sk_destruct = sock_def_destruct; 2793 2794 sk->sk_frag.page = NULL; 2795 sk->sk_frag.offset = 0; 2796 sk->sk_peek_off = -1; 2797 2798 sk->sk_peer_pid = NULL; 2799 sk->sk_peer_cred = NULL; 2800 sk->sk_write_pending = 0; 2801 sk->sk_rcvlowat = 1; 2802 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 2803 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 2804 2805 sk->sk_stamp = SK_DEFAULT_STAMP; 2806 atomic_set(&sk->sk_zckey, 0); 2807 2808 #ifdef CONFIG_NET_RX_BUSY_POLL 2809 sk->sk_napi_id = 0; 2810 sk->sk_ll_usec = sysctl_net_busy_read; 2811 #endif 2812 2813 sk->sk_max_pacing_rate = ~0U; 2814 sk->sk_pacing_rate = ~0U; 2815 sk->sk_pacing_shift = 10; 2816 sk->sk_incoming_cpu = -1; 2817 2818 sk_rx_queue_clear(sk); 2819 /* 2820 * Before updating sk_refcnt, we must commit prior changes to memory 2821 * (Documentation/RCU/rculist_nulls.txt for details) 2822 */ 2823 smp_wmb(); 2824 refcount_set(&sk->sk_refcnt, 1); 2825 atomic_set(&sk->sk_drops, 0); 2826 } 2827 EXPORT_SYMBOL(sock_init_data); 2828 2829 void lock_sock_nested(struct sock *sk, int subclass) 2830 { 2831 might_sleep(); 2832 spin_lock_bh(&sk->sk_lock.slock); 2833 if (sk->sk_lock.owned) 2834 __lock_sock(sk); 2835 sk->sk_lock.owned = 1; 2836 spin_unlock(&sk->sk_lock.slock); 2837 /* 2838 * The sk_lock has mutex_lock() semantics here: 2839 */ 2840 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 2841 local_bh_enable(); 2842 } 2843 EXPORT_SYMBOL(lock_sock_nested); 2844 2845 void release_sock(struct sock *sk) 2846 { 2847 spin_lock_bh(&sk->sk_lock.slock); 2848 if (sk->sk_backlog.tail) 2849 __release_sock(sk); 2850 2851 /* Warning : release_cb() might need to release sk ownership, 2852 * ie call sock_release_ownership(sk) before us. 2853 */ 2854 if (sk->sk_prot->release_cb) 2855 sk->sk_prot->release_cb(sk); 2856 2857 sock_release_ownership(sk); 2858 if (waitqueue_active(&sk->sk_lock.wq)) 2859 wake_up(&sk->sk_lock.wq); 2860 spin_unlock_bh(&sk->sk_lock.slock); 2861 } 2862 EXPORT_SYMBOL(release_sock); 2863 2864 /** 2865 * lock_sock_fast - fast version of lock_sock 2866 * @sk: socket 2867 * 2868 * This version should be used for very small section, where process wont block 2869 * return false if fast path is taken: 2870 * 2871 * sk_lock.slock locked, owned = 0, BH disabled 2872 * 2873 * return true if slow path is taken: 2874 * 2875 * sk_lock.slock unlocked, owned = 1, BH enabled 2876 */ 2877 bool lock_sock_fast(struct sock *sk) 2878 { 2879 might_sleep(); 2880 spin_lock_bh(&sk->sk_lock.slock); 2881 2882 if (!sk->sk_lock.owned) 2883 /* 2884 * Note : We must disable BH 2885 */ 2886 return false; 2887 2888 __lock_sock(sk); 2889 sk->sk_lock.owned = 1; 2890 spin_unlock(&sk->sk_lock.slock); 2891 /* 2892 * The sk_lock has mutex_lock() semantics here: 2893 */ 2894 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_); 2895 local_bh_enable(); 2896 return true; 2897 } 2898 EXPORT_SYMBOL(lock_sock_fast); 2899 2900 int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp) 2901 { 2902 struct timeval tv; 2903 2904 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2905 tv = ktime_to_timeval(sk->sk_stamp); 2906 if (tv.tv_sec == -1) 2907 return -ENOENT; 2908 if (tv.tv_sec == 0) { 2909 sk->sk_stamp = ktime_get_real(); 2910 tv = ktime_to_timeval(sk->sk_stamp); 2911 } 2912 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0; 2913 } 2914 EXPORT_SYMBOL(sock_get_timestamp); 2915 2916 int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp) 2917 { 2918 struct timespec ts; 2919 2920 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2921 ts = ktime_to_timespec(sk->sk_stamp); 2922 if (ts.tv_sec == -1) 2923 return -ENOENT; 2924 if (ts.tv_sec == 0) { 2925 sk->sk_stamp = ktime_get_real(); 2926 ts = ktime_to_timespec(sk->sk_stamp); 2927 } 2928 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0; 2929 } 2930 EXPORT_SYMBOL(sock_get_timestampns); 2931 2932 void sock_enable_timestamp(struct sock *sk, int flag) 2933 { 2934 if (!sock_flag(sk, flag)) { 2935 unsigned long previous_flags = sk->sk_flags; 2936 2937 sock_set_flag(sk, flag); 2938 /* 2939 * we just set one of the two flags which require net 2940 * time stamping, but time stamping might have been on 2941 * already because of the other one 2942 */ 2943 if (sock_needs_netstamp(sk) && 2944 !(previous_flags & SK_FLAGS_TIMESTAMP)) 2945 net_enable_timestamp(); 2946 } 2947 } 2948 2949 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 2950 int level, int type) 2951 { 2952 struct sock_exterr_skb *serr; 2953 struct sk_buff *skb; 2954 int copied, err; 2955 2956 err = -EAGAIN; 2957 skb = sock_dequeue_err_skb(sk); 2958 if (skb == NULL) 2959 goto out; 2960 2961 copied = skb->len; 2962 if (copied > len) { 2963 msg->msg_flags |= MSG_TRUNC; 2964 copied = len; 2965 } 2966 err = skb_copy_datagram_msg(skb, 0, msg, copied); 2967 if (err) 2968 goto out_free_skb; 2969 2970 sock_recv_timestamp(msg, sk, skb); 2971 2972 serr = SKB_EXT_ERR(skb); 2973 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 2974 2975 msg->msg_flags |= MSG_ERRQUEUE; 2976 err = copied; 2977 2978 out_free_skb: 2979 kfree_skb(skb); 2980 out: 2981 return err; 2982 } 2983 EXPORT_SYMBOL(sock_recv_errqueue); 2984 2985 /* 2986 * Get a socket option on an socket. 2987 * 2988 * FIX: POSIX 1003.1g is very ambiguous here. It states that 2989 * asynchronous errors should be reported by getsockopt. We assume 2990 * this means if you specify SO_ERROR (otherwise whats the point of it). 2991 */ 2992 int sock_common_getsockopt(struct socket *sock, int level, int optname, 2993 char __user *optval, int __user *optlen) 2994 { 2995 struct sock *sk = sock->sk; 2996 2997 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 2998 } 2999 EXPORT_SYMBOL(sock_common_getsockopt); 3000 3001 #ifdef CONFIG_COMPAT 3002 int compat_sock_common_getsockopt(struct socket *sock, int level, int optname, 3003 char __user *optval, int __user *optlen) 3004 { 3005 struct sock *sk = sock->sk; 3006 3007 if (sk->sk_prot->compat_getsockopt != NULL) 3008 return sk->sk_prot->compat_getsockopt(sk, level, optname, 3009 optval, optlen); 3010 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 3011 } 3012 EXPORT_SYMBOL(compat_sock_common_getsockopt); 3013 #endif 3014 3015 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 3016 int flags) 3017 { 3018 struct sock *sk = sock->sk; 3019 int addr_len = 0; 3020 int err; 3021 3022 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, 3023 flags & ~MSG_DONTWAIT, &addr_len); 3024 if (err >= 0) 3025 msg->msg_namelen = addr_len; 3026 return err; 3027 } 3028 EXPORT_SYMBOL(sock_common_recvmsg); 3029 3030 /* 3031 * Set socket options on an inet socket. 3032 */ 3033 int sock_common_setsockopt(struct socket *sock, int level, int optname, 3034 char __user *optval, unsigned int optlen) 3035 { 3036 struct sock *sk = sock->sk; 3037 3038 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3039 } 3040 EXPORT_SYMBOL(sock_common_setsockopt); 3041 3042 #ifdef CONFIG_COMPAT 3043 int compat_sock_common_setsockopt(struct socket *sock, int level, int optname, 3044 char __user *optval, unsigned int optlen) 3045 { 3046 struct sock *sk = sock->sk; 3047 3048 if (sk->sk_prot->compat_setsockopt != NULL) 3049 return sk->sk_prot->compat_setsockopt(sk, level, optname, 3050 optval, optlen); 3051 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3052 } 3053 EXPORT_SYMBOL(compat_sock_common_setsockopt); 3054 #endif 3055 3056 void sk_common_release(struct sock *sk) 3057 { 3058 if (sk->sk_prot->destroy) 3059 sk->sk_prot->destroy(sk); 3060 3061 /* 3062 * Observation: when sock_common_release is called, processes have 3063 * no access to socket. But net still has. 3064 * Step one, detach it from networking: 3065 * 3066 * A. Remove from hash tables. 3067 */ 3068 3069 sk->sk_prot->unhash(sk); 3070 3071 /* 3072 * In this point socket cannot receive new packets, but it is possible 3073 * that some packets are in flight because some CPU runs receiver and 3074 * did hash table lookup before we unhashed socket. They will achieve 3075 * receive queue and will be purged by socket destructor. 3076 * 3077 * Also we still have packets pending on receive queue and probably, 3078 * our own packets waiting in device queues. sock_destroy will drain 3079 * receive queue, but transmitted packets will delay socket destruction 3080 * until the last reference will be released. 3081 */ 3082 3083 sock_orphan(sk); 3084 3085 xfrm_sk_free_policy(sk); 3086 3087 sk_refcnt_debug_release(sk); 3088 3089 sock_put(sk); 3090 } 3091 EXPORT_SYMBOL(sk_common_release); 3092 3093 void sk_get_meminfo(const struct sock *sk, u32 *mem) 3094 { 3095 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 3096 3097 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 3098 mem[SK_MEMINFO_RCVBUF] = sk->sk_rcvbuf; 3099 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 3100 mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf; 3101 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc; 3102 mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued; 3103 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 3104 mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len; 3105 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops); 3106 } 3107 3108 #ifdef CONFIG_PROC_FS 3109 #define PROTO_INUSE_NR 64 /* should be enough for the first time */ 3110 struct prot_inuse { 3111 int val[PROTO_INUSE_NR]; 3112 }; 3113 3114 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 3115 3116 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) 3117 { 3118 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); 3119 } 3120 EXPORT_SYMBOL_GPL(sock_prot_inuse_add); 3121 3122 int sock_prot_inuse_get(struct net *net, struct proto *prot) 3123 { 3124 int cpu, idx = prot->inuse_idx; 3125 int res = 0; 3126 3127 for_each_possible_cpu(cpu) 3128 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 3129 3130 return res >= 0 ? res : 0; 3131 } 3132 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 3133 3134 static void sock_inuse_add(struct net *net, int val) 3135 { 3136 this_cpu_add(*net->core.sock_inuse, val); 3137 } 3138 3139 int sock_inuse_get(struct net *net) 3140 { 3141 int cpu, res = 0; 3142 3143 for_each_possible_cpu(cpu) 3144 res += *per_cpu_ptr(net->core.sock_inuse, cpu); 3145 3146 return res; 3147 } 3148 3149 EXPORT_SYMBOL_GPL(sock_inuse_get); 3150 3151 static int __net_init sock_inuse_init_net(struct net *net) 3152 { 3153 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 3154 if (net->core.prot_inuse == NULL) 3155 return -ENOMEM; 3156 3157 net->core.sock_inuse = alloc_percpu(int); 3158 if (net->core.sock_inuse == NULL) 3159 goto out; 3160 3161 return 0; 3162 3163 out: 3164 free_percpu(net->core.prot_inuse); 3165 return -ENOMEM; 3166 } 3167 3168 static void __net_exit sock_inuse_exit_net(struct net *net) 3169 { 3170 free_percpu(net->core.prot_inuse); 3171 free_percpu(net->core.sock_inuse); 3172 } 3173 3174 static struct pernet_operations net_inuse_ops = { 3175 .init = sock_inuse_init_net, 3176 .exit = sock_inuse_exit_net, 3177 }; 3178 3179 static __init int net_inuse_init(void) 3180 { 3181 if (register_pernet_subsys(&net_inuse_ops)) 3182 panic("Cannot initialize net inuse counters"); 3183 3184 return 0; 3185 } 3186 3187 core_initcall(net_inuse_init); 3188 3189 static void assign_proto_idx(struct proto *prot) 3190 { 3191 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 3192 3193 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 3194 pr_err("PROTO_INUSE_NR exhausted\n"); 3195 return; 3196 } 3197 3198 set_bit(prot->inuse_idx, proto_inuse_idx); 3199 } 3200 3201 static void release_proto_idx(struct proto *prot) 3202 { 3203 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 3204 clear_bit(prot->inuse_idx, proto_inuse_idx); 3205 } 3206 #else 3207 static inline void assign_proto_idx(struct proto *prot) 3208 { 3209 } 3210 3211 static inline void release_proto_idx(struct proto *prot) 3212 { 3213 } 3214 3215 static void sock_inuse_add(struct net *net, int val) 3216 { 3217 } 3218 #endif 3219 3220 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 3221 { 3222 if (!rsk_prot) 3223 return; 3224 kfree(rsk_prot->slab_name); 3225 rsk_prot->slab_name = NULL; 3226 kmem_cache_destroy(rsk_prot->slab); 3227 rsk_prot->slab = NULL; 3228 } 3229 3230 static int req_prot_init(const struct proto *prot) 3231 { 3232 struct request_sock_ops *rsk_prot = prot->rsk_prot; 3233 3234 if (!rsk_prot) 3235 return 0; 3236 3237 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 3238 prot->name); 3239 if (!rsk_prot->slab_name) 3240 return -ENOMEM; 3241 3242 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 3243 rsk_prot->obj_size, 0, 3244 SLAB_ACCOUNT | prot->slab_flags, 3245 NULL); 3246 3247 if (!rsk_prot->slab) { 3248 pr_crit("%s: Can't create request sock SLAB cache!\n", 3249 prot->name); 3250 return -ENOMEM; 3251 } 3252 return 0; 3253 } 3254 3255 int proto_register(struct proto *prot, int alloc_slab) 3256 { 3257 if (alloc_slab) { 3258 prot->slab = kmem_cache_create_usercopy(prot->name, 3259 prot->obj_size, 0, 3260 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 3261 prot->slab_flags, 3262 prot->useroffset, prot->usersize, 3263 NULL); 3264 3265 if (prot->slab == NULL) { 3266 pr_crit("%s: Can't create sock SLAB cache!\n", 3267 prot->name); 3268 goto out; 3269 } 3270 3271 if (req_prot_init(prot)) 3272 goto out_free_request_sock_slab; 3273 3274 if (prot->twsk_prot != NULL) { 3275 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name); 3276 3277 if (prot->twsk_prot->twsk_slab_name == NULL) 3278 goto out_free_request_sock_slab; 3279 3280 prot->twsk_prot->twsk_slab = 3281 kmem_cache_create(prot->twsk_prot->twsk_slab_name, 3282 prot->twsk_prot->twsk_obj_size, 3283 0, 3284 SLAB_ACCOUNT | 3285 prot->slab_flags, 3286 NULL); 3287 if (prot->twsk_prot->twsk_slab == NULL) 3288 goto out_free_timewait_sock_slab_name; 3289 } 3290 } 3291 3292 mutex_lock(&proto_list_mutex); 3293 list_add(&prot->node, &proto_list); 3294 assign_proto_idx(prot); 3295 mutex_unlock(&proto_list_mutex); 3296 return 0; 3297 3298 out_free_timewait_sock_slab_name: 3299 kfree(prot->twsk_prot->twsk_slab_name); 3300 out_free_request_sock_slab: 3301 req_prot_cleanup(prot->rsk_prot); 3302 3303 kmem_cache_destroy(prot->slab); 3304 prot->slab = NULL; 3305 out: 3306 return -ENOBUFS; 3307 } 3308 EXPORT_SYMBOL(proto_register); 3309 3310 void proto_unregister(struct proto *prot) 3311 { 3312 mutex_lock(&proto_list_mutex); 3313 release_proto_idx(prot); 3314 list_del(&prot->node); 3315 mutex_unlock(&proto_list_mutex); 3316 3317 kmem_cache_destroy(prot->slab); 3318 prot->slab = NULL; 3319 3320 req_prot_cleanup(prot->rsk_prot); 3321 3322 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) { 3323 kmem_cache_destroy(prot->twsk_prot->twsk_slab); 3324 kfree(prot->twsk_prot->twsk_slab_name); 3325 prot->twsk_prot->twsk_slab = NULL; 3326 } 3327 } 3328 EXPORT_SYMBOL(proto_unregister); 3329 3330 int sock_load_diag_module(int family, int protocol) 3331 { 3332 if (!protocol) { 3333 if (!sock_is_registered(family)) 3334 return -ENOENT; 3335 3336 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 3337 NETLINK_SOCK_DIAG, family); 3338 } 3339 3340 #ifdef CONFIG_INET 3341 if (family == AF_INET && 3342 !rcu_access_pointer(inet_protos[protocol])) 3343 return -ENOENT; 3344 #endif 3345 3346 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 3347 NETLINK_SOCK_DIAG, family, protocol); 3348 } 3349 EXPORT_SYMBOL(sock_load_diag_module); 3350 3351 #ifdef CONFIG_PROC_FS 3352 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 3353 __acquires(proto_list_mutex) 3354 { 3355 mutex_lock(&proto_list_mutex); 3356 return seq_list_start_head(&proto_list, *pos); 3357 } 3358 3359 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3360 { 3361 return seq_list_next(v, &proto_list, pos); 3362 } 3363 3364 static void proto_seq_stop(struct seq_file *seq, void *v) 3365 __releases(proto_list_mutex) 3366 { 3367 mutex_unlock(&proto_list_mutex); 3368 } 3369 3370 static char proto_method_implemented(const void *method) 3371 { 3372 return method == NULL ? 'n' : 'y'; 3373 } 3374 static long sock_prot_memory_allocated(struct proto *proto) 3375 { 3376 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 3377 } 3378 3379 static char *sock_prot_memory_pressure(struct proto *proto) 3380 { 3381 return proto->memory_pressure != NULL ? 3382 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 3383 } 3384 3385 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 3386 { 3387 3388 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 3389 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 3390 proto->name, 3391 proto->obj_size, 3392 sock_prot_inuse_get(seq_file_net(seq), proto), 3393 sock_prot_memory_allocated(proto), 3394 sock_prot_memory_pressure(proto), 3395 proto->max_header, 3396 proto->slab == NULL ? "no" : "yes", 3397 module_name(proto->owner), 3398 proto_method_implemented(proto->close), 3399 proto_method_implemented(proto->connect), 3400 proto_method_implemented(proto->disconnect), 3401 proto_method_implemented(proto->accept), 3402 proto_method_implemented(proto->ioctl), 3403 proto_method_implemented(proto->init), 3404 proto_method_implemented(proto->destroy), 3405 proto_method_implemented(proto->shutdown), 3406 proto_method_implemented(proto->setsockopt), 3407 proto_method_implemented(proto->getsockopt), 3408 proto_method_implemented(proto->sendmsg), 3409 proto_method_implemented(proto->recvmsg), 3410 proto_method_implemented(proto->sendpage), 3411 proto_method_implemented(proto->bind), 3412 proto_method_implemented(proto->backlog_rcv), 3413 proto_method_implemented(proto->hash), 3414 proto_method_implemented(proto->unhash), 3415 proto_method_implemented(proto->get_port), 3416 proto_method_implemented(proto->enter_memory_pressure)); 3417 } 3418 3419 static int proto_seq_show(struct seq_file *seq, void *v) 3420 { 3421 if (v == &proto_list) 3422 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 3423 "protocol", 3424 "size", 3425 "sockets", 3426 "memory", 3427 "press", 3428 "maxhdr", 3429 "slab", 3430 "module", 3431 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n"); 3432 else 3433 proto_seq_printf(seq, list_entry(v, struct proto, node)); 3434 return 0; 3435 } 3436 3437 static const struct seq_operations proto_seq_ops = { 3438 .start = proto_seq_start, 3439 .next = proto_seq_next, 3440 .stop = proto_seq_stop, 3441 .show = proto_seq_show, 3442 }; 3443 3444 static __net_init int proto_init_net(struct net *net) 3445 { 3446 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 3447 sizeof(struct seq_net_private))) 3448 return -ENOMEM; 3449 3450 return 0; 3451 } 3452 3453 static __net_exit void proto_exit_net(struct net *net) 3454 { 3455 remove_proc_entry("protocols", net->proc_net); 3456 } 3457 3458 3459 static __net_initdata struct pernet_operations proto_net_ops = { 3460 .init = proto_init_net, 3461 .exit = proto_exit_net, 3462 }; 3463 3464 static int __init proto_init(void) 3465 { 3466 return register_pernet_subsys(&proto_net_ops); 3467 } 3468 3469 subsys_initcall(proto_init); 3470 3471 #endif /* PROC_FS */ 3472 3473 #ifdef CONFIG_NET_RX_BUSY_POLL 3474 bool sk_busy_loop_end(void *p, unsigned long start_time) 3475 { 3476 struct sock *sk = p; 3477 3478 return !skb_queue_empty(&sk->sk_receive_queue) || 3479 sk_busy_loop_timeout(sk, start_time); 3480 } 3481 EXPORT_SYMBOL(sk_busy_loop_end); 3482 #endif /* CONFIG_NET_RX_BUSY_POLL */ 3483