1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Generic socket support routines. Memory allocators, socket lock/release 7 * handler for protocols to use and generic option handler. 8 * 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 * 85 * 86 * This program is free software; you can redistribute it and/or 87 * modify it under the terms of the GNU General Public License 88 * as published by the Free Software Foundation; either version 89 * 2 of the License, or (at your option) any later version. 90 */ 91 92 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 93 94 #include <asm/unaligned.h> 95 #include <linux/capability.h> 96 #include <linux/errno.h> 97 #include <linux/errqueue.h> 98 #include <linux/types.h> 99 #include <linux/socket.h> 100 #include <linux/in.h> 101 #include <linux/kernel.h> 102 #include <linux/module.h> 103 #include <linux/proc_fs.h> 104 #include <linux/seq_file.h> 105 #include <linux/sched.h> 106 #include <linux/sched/mm.h> 107 #include <linux/timer.h> 108 #include <linux/string.h> 109 #include <linux/sockios.h> 110 #include <linux/net.h> 111 #include <linux/mm.h> 112 #include <linux/slab.h> 113 #include <linux/interrupt.h> 114 #include <linux/poll.h> 115 #include <linux/tcp.h> 116 #include <linux/init.h> 117 #include <linux/highmem.h> 118 #include <linux/user_namespace.h> 119 #include <linux/static_key.h> 120 #include <linux/memcontrol.h> 121 #include <linux/prefetch.h> 122 123 #include <linux/uaccess.h> 124 125 #include <linux/netdevice.h> 126 #include <net/protocol.h> 127 #include <linux/skbuff.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <linux/net_tstamp.h> 132 #include <net/xfrm.h> 133 #include <linux/ipsec.h> 134 #include <net/cls_cgroup.h> 135 #include <net/netprio_cgroup.h> 136 #include <linux/sock_diag.h> 137 138 #include <linux/filter.h> 139 #include <net/sock_reuseport.h> 140 141 #include <trace/events/sock.h> 142 143 #include <net/tcp.h> 144 #include <net/busy_poll.h> 145 146 static DEFINE_MUTEX(proto_list_mutex); 147 static LIST_HEAD(proto_list); 148 149 static void sock_inuse_add(struct net *net, int val); 150 151 /** 152 * sk_ns_capable - General socket capability test 153 * @sk: Socket to use a capability on or through 154 * @user_ns: The user namespace of the capability to use 155 * @cap: The capability to use 156 * 157 * Test to see if the opener of the socket had when the socket was 158 * created and the current process has the capability @cap in the user 159 * namespace @user_ns. 160 */ 161 bool sk_ns_capable(const struct sock *sk, 162 struct user_namespace *user_ns, int cap) 163 { 164 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 165 ns_capable(user_ns, cap); 166 } 167 EXPORT_SYMBOL(sk_ns_capable); 168 169 /** 170 * sk_capable - Socket global capability test 171 * @sk: Socket to use a capability on or through 172 * @cap: The global capability to use 173 * 174 * Test to see if the opener of the socket had when the socket was 175 * created and the current process has the capability @cap in all user 176 * namespaces. 177 */ 178 bool sk_capable(const struct sock *sk, int cap) 179 { 180 return sk_ns_capable(sk, &init_user_ns, cap); 181 } 182 EXPORT_SYMBOL(sk_capable); 183 184 /** 185 * sk_net_capable - Network namespace socket capability test 186 * @sk: Socket to use a capability on or through 187 * @cap: The capability to use 188 * 189 * Test to see if the opener of the socket had when the socket was created 190 * and the current process has the capability @cap over the network namespace 191 * the socket is a member of. 192 */ 193 bool sk_net_capable(const struct sock *sk, int cap) 194 { 195 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 196 } 197 EXPORT_SYMBOL(sk_net_capable); 198 199 /* 200 * Each address family might have different locking rules, so we have 201 * one slock key per address family and separate keys for internal and 202 * userspace sockets. 203 */ 204 static struct lock_class_key af_family_keys[AF_MAX]; 205 static struct lock_class_key af_family_kern_keys[AF_MAX]; 206 static struct lock_class_key af_family_slock_keys[AF_MAX]; 207 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 208 209 /* 210 * Make lock validator output more readable. (we pre-construct these 211 * strings build-time, so that runtime initialization of socket 212 * locks is fast): 213 */ 214 215 #define _sock_locks(x) \ 216 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 217 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 218 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 219 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 220 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 221 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 222 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 223 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 224 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 225 x "27" , x "28" , x "AF_CAN" , \ 226 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 227 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 228 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 229 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 230 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 231 x "AF_MAX" 232 233 static const char *const af_family_key_strings[AF_MAX+1] = { 234 _sock_locks("sk_lock-") 235 }; 236 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 237 _sock_locks("slock-") 238 }; 239 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 240 _sock_locks("clock-") 241 }; 242 243 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 244 _sock_locks("k-sk_lock-") 245 }; 246 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 247 _sock_locks("k-slock-") 248 }; 249 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 250 _sock_locks("k-clock-") 251 }; 252 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 253 _sock_locks("rlock-") 254 }; 255 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 256 _sock_locks("wlock-") 257 }; 258 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 259 _sock_locks("elock-") 260 }; 261 262 /* 263 * sk_callback_lock and sk queues locking rules are per-address-family, 264 * so split the lock classes by using a per-AF key: 265 */ 266 static struct lock_class_key af_callback_keys[AF_MAX]; 267 static struct lock_class_key af_rlock_keys[AF_MAX]; 268 static struct lock_class_key af_wlock_keys[AF_MAX]; 269 static struct lock_class_key af_elock_keys[AF_MAX]; 270 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 271 272 /* Run time adjustable parameters. */ 273 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 274 EXPORT_SYMBOL(sysctl_wmem_max); 275 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 276 EXPORT_SYMBOL(sysctl_rmem_max); 277 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 278 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 279 280 /* Maximal space eaten by iovec or ancillary data plus some space */ 281 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 282 EXPORT_SYMBOL(sysctl_optmem_max); 283 284 int sysctl_tstamp_allow_data __read_mostly = 1; 285 286 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 287 EXPORT_SYMBOL_GPL(memalloc_socks_key); 288 289 /** 290 * sk_set_memalloc - sets %SOCK_MEMALLOC 291 * @sk: socket to set it on 292 * 293 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 294 * It's the responsibility of the admin to adjust min_free_kbytes 295 * to meet the requirements 296 */ 297 void sk_set_memalloc(struct sock *sk) 298 { 299 sock_set_flag(sk, SOCK_MEMALLOC); 300 sk->sk_allocation |= __GFP_MEMALLOC; 301 static_branch_inc(&memalloc_socks_key); 302 } 303 EXPORT_SYMBOL_GPL(sk_set_memalloc); 304 305 void sk_clear_memalloc(struct sock *sk) 306 { 307 sock_reset_flag(sk, SOCK_MEMALLOC); 308 sk->sk_allocation &= ~__GFP_MEMALLOC; 309 static_branch_dec(&memalloc_socks_key); 310 311 /* 312 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 313 * progress of swapping. SOCK_MEMALLOC may be cleared while 314 * it has rmem allocations due to the last swapfile being deactivated 315 * but there is a risk that the socket is unusable due to exceeding 316 * the rmem limits. Reclaim the reserves and obey rmem limits again. 317 */ 318 sk_mem_reclaim(sk); 319 } 320 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 321 322 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 323 { 324 int ret; 325 unsigned int noreclaim_flag; 326 327 /* these should have been dropped before queueing */ 328 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 329 330 noreclaim_flag = memalloc_noreclaim_save(); 331 ret = sk->sk_backlog_rcv(sk, skb); 332 memalloc_noreclaim_restore(noreclaim_flag); 333 334 return ret; 335 } 336 EXPORT_SYMBOL(__sk_backlog_rcv); 337 338 static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen) 339 { 340 struct timeval tv; 341 342 if (optlen < sizeof(tv)) 343 return -EINVAL; 344 if (copy_from_user(&tv, optval, sizeof(tv))) 345 return -EFAULT; 346 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 347 return -EDOM; 348 349 if (tv.tv_sec < 0) { 350 static int warned __read_mostly; 351 352 *timeo_p = 0; 353 if (warned < 10 && net_ratelimit()) { 354 warned++; 355 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 356 __func__, current->comm, task_pid_nr(current)); 357 } 358 return 0; 359 } 360 *timeo_p = MAX_SCHEDULE_TIMEOUT; 361 if (tv.tv_sec == 0 && tv.tv_usec == 0) 362 return 0; 363 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1)) 364 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ); 365 return 0; 366 } 367 368 static void sock_warn_obsolete_bsdism(const char *name) 369 { 370 static int warned; 371 static char warncomm[TASK_COMM_LEN]; 372 if (strcmp(warncomm, current->comm) && warned < 5) { 373 strcpy(warncomm, current->comm); 374 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n", 375 warncomm, name); 376 warned++; 377 } 378 } 379 380 static bool sock_needs_netstamp(const struct sock *sk) 381 { 382 switch (sk->sk_family) { 383 case AF_UNSPEC: 384 case AF_UNIX: 385 return false; 386 default: 387 return true; 388 } 389 } 390 391 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 392 { 393 if (sk->sk_flags & flags) { 394 sk->sk_flags &= ~flags; 395 if (sock_needs_netstamp(sk) && 396 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 397 net_disable_timestamp(); 398 } 399 } 400 401 402 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 403 { 404 unsigned long flags; 405 struct sk_buff_head *list = &sk->sk_receive_queue; 406 407 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { 408 atomic_inc(&sk->sk_drops); 409 trace_sock_rcvqueue_full(sk, skb); 410 return -ENOMEM; 411 } 412 413 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 414 atomic_inc(&sk->sk_drops); 415 return -ENOBUFS; 416 } 417 418 skb->dev = NULL; 419 skb_set_owner_r(skb, sk); 420 421 /* we escape from rcu protected region, make sure we dont leak 422 * a norefcounted dst 423 */ 424 skb_dst_force(skb); 425 426 spin_lock_irqsave(&list->lock, flags); 427 sock_skb_set_dropcount(sk, skb); 428 __skb_queue_tail(list, skb); 429 spin_unlock_irqrestore(&list->lock, flags); 430 431 if (!sock_flag(sk, SOCK_DEAD)) 432 sk->sk_data_ready(sk); 433 return 0; 434 } 435 EXPORT_SYMBOL(__sock_queue_rcv_skb); 436 437 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 438 { 439 int err; 440 441 err = sk_filter(sk, skb); 442 if (err) 443 return err; 444 445 return __sock_queue_rcv_skb(sk, skb); 446 } 447 EXPORT_SYMBOL(sock_queue_rcv_skb); 448 449 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 450 const int nested, unsigned int trim_cap, bool refcounted) 451 { 452 int rc = NET_RX_SUCCESS; 453 454 if (sk_filter_trim_cap(sk, skb, trim_cap)) 455 goto discard_and_relse; 456 457 skb->dev = NULL; 458 459 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) { 460 atomic_inc(&sk->sk_drops); 461 goto discard_and_relse; 462 } 463 if (nested) 464 bh_lock_sock_nested(sk); 465 else 466 bh_lock_sock(sk); 467 if (!sock_owned_by_user(sk)) { 468 /* 469 * trylock + unlock semantics: 470 */ 471 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 472 473 rc = sk_backlog_rcv(sk, skb); 474 475 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_); 476 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) { 477 bh_unlock_sock(sk); 478 atomic_inc(&sk->sk_drops); 479 goto discard_and_relse; 480 } 481 482 bh_unlock_sock(sk); 483 out: 484 if (refcounted) 485 sock_put(sk); 486 return rc; 487 discard_and_relse: 488 kfree_skb(skb); 489 goto out; 490 } 491 EXPORT_SYMBOL(__sk_receive_skb); 492 493 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 494 { 495 struct dst_entry *dst = __sk_dst_get(sk); 496 497 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 498 sk_tx_queue_clear(sk); 499 sk->sk_dst_pending_confirm = 0; 500 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 501 dst_release(dst); 502 return NULL; 503 } 504 505 return dst; 506 } 507 EXPORT_SYMBOL(__sk_dst_check); 508 509 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 510 { 511 struct dst_entry *dst = sk_dst_get(sk); 512 513 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 514 sk_dst_reset(sk); 515 dst_release(dst); 516 return NULL; 517 } 518 519 return dst; 520 } 521 EXPORT_SYMBOL(sk_dst_check); 522 523 static int sock_setbindtodevice(struct sock *sk, char __user *optval, 524 int optlen) 525 { 526 int ret = -ENOPROTOOPT; 527 #ifdef CONFIG_NETDEVICES 528 struct net *net = sock_net(sk); 529 char devname[IFNAMSIZ]; 530 int index; 531 532 /* Sorry... */ 533 ret = -EPERM; 534 if (!ns_capable(net->user_ns, CAP_NET_RAW)) 535 goto out; 536 537 ret = -EINVAL; 538 if (optlen < 0) 539 goto out; 540 541 /* Bind this socket to a particular device like "eth0", 542 * as specified in the passed interface name. If the 543 * name is "" or the option length is zero the socket 544 * is not bound. 545 */ 546 if (optlen > IFNAMSIZ - 1) 547 optlen = IFNAMSIZ - 1; 548 memset(devname, 0, sizeof(devname)); 549 550 ret = -EFAULT; 551 if (copy_from_user(devname, optval, optlen)) 552 goto out; 553 554 index = 0; 555 if (devname[0] != '\0') { 556 struct net_device *dev; 557 558 rcu_read_lock(); 559 dev = dev_get_by_name_rcu(net, devname); 560 if (dev) 561 index = dev->ifindex; 562 rcu_read_unlock(); 563 ret = -ENODEV; 564 if (!dev) 565 goto out; 566 } 567 568 lock_sock(sk); 569 sk->sk_bound_dev_if = index; 570 sk_dst_reset(sk); 571 release_sock(sk); 572 573 ret = 0; 574 575 out: 576 #endif 577 578 return ret; 579 } 580 581 static int sock_getbindtodevice(struct sock *sk, char __user *optval, 582 int __user *optlen, int len) 583 { 584 int ret = -ENOPROTOOPT; 585 #ifdef CONFIG_NETDEVICES 586 struct net *net = sock_net(sk); 587 char devname[IFNAMSIZ]; 588 589 if (sk->sk_bound_dev_if == 0) { 590 len = 0; 591 goto zero; 592 } 593 594 ret = -EINVAL; 595 if (len < IFNAMSIZ) 596 goto out; 597 598 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if); 599 if (ret) 600 goto out; 601 602 len = strlen(devname) + 1; 603 604 ret = -EFAULT; 605 if (copy_to_user(optval, devname, len)) 606 goto out; 607 608 zero: 609 ret = -EFAULT; 610 if (put_user(len, optlen)) 611 goto out; 612 613 ret = 0; 614 615 out: 616 #endif 617 618 return ret; 619 } 620 621 static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool) 622 { 623 if (valbool) 624 sock_set_flag(sk, bit); 625 else 626 sock_reset_flag(sk, bit); 627 } 628 629 bool sk_mc_loop(struct sock *sk) 630 { 631 if (dev_recursion_level()) 632 return false; 633 if (!sk) 634 return true; 635 switch (sk->sk_family) { 636 case AF_INET: 637 return inet_sk(sk)->mc_loop; 638 #if IS_ENABLED(CONFIG_IPV6) 639 case AF_INET6: 640 return inet6_sk(sk)->mc_loop; 641 #endif 642 } 643 WARN_ON(1); 644 return true; 645 } 646 EXPORT_SYMBOL(sk_mc_loop); 647 648 /* 649 * This is meant for all protocols to use and covers goings on 650 * at the socket level. Everything here is generic. 651 */ 652 653 int sock_setsockopt(struct socket *sock, int level, int optname, 654 char __user *optval, unsigned int optlen) 655 { 656 struct sock_txtime sk_txtime; 657 struct sock *sk = sock->sk; 658 int val; 659 int valbool; 660 struct linger ling; 661 int ret = 0; 662 663 /* 664 * Options without arguments 665 */ 666 667 if (optname == SO_BINDTODEVICE) 668 return sock_setbindtodevice(sk, optval, optlen); 669 670 if (optlen < sizeof(int)) 671 return -EINVAL; 672 673 if (get_user(val, (int __user *)optval)) 674 return -EFAULT; 675 676 valbool = val ? 1 : 0; 677 678 lock_sock(sk); 679 680 switch (optname) { 681 case SO_DEBUG: 682 if (val && !capable(CAP_NET_ADMIN)) 683 ret = -EACCES; 684 else 685 sock_valbool_flag(sk, SOCK_DBG, valbool); 686 break; 687 case SO_REUSEADDR: 688 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 689 break; 690 case SO_REUSEPORT: 691 sk->sk_reuseport = valbool; 692 break; 693 case SO_TYPE: 694 case SO_PROTOCOL: 695 case SO_DOMAIN: 696 case SO_ERROR: 697 ret = -ENOPROTOOPT; 698 break; 699 case SO_DONTROUTE: 700 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 701 break; 702 case SO_BROADCAST: 703 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 704 break; 705 case SO_SNDBUF: 706 /* Don't error on this BSD doesn't and if you think 707 * about it this is right. Otherwise apps have to 708 * play 'guess the biggest size' games. RCVBUF/SNDBUF 709 * are treated in BSD as hints 710 */ 711 val = min_t(u32, val, sysctl_wmem_max); 712 set_sndbuf: 713 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 714 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); 715 /* Wake up sending tasks if we upped the value. */ 716 sk->sk_write_space(sk); 717 break; 718 719 case SO_SNDBUFFORCE: 720 if (!capable(CAP_NET_ADMIN)) { 721 ret = -EPERM; 722 break; 723 } 724 goto set_sndbuf; 725 726 case SO_RCVBUF: 727 /* Don't error on this BSD doesn't and if you think 728 * about it this is right. Otherwise apps have to 729 * play 'guess the biggest size' games. RCVBUF/SNDBUF 730 * are treated in BSD as hints 731 */ 732 val = min_t(u32, val, sysctl_rmem_max); 733 set_rcvbuf: 734 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 735 /* 736 * We double it on the way in to account for 737 * "struct sk_buff" etc. overhead. Applications 738 * assume that the SO_RCVBUF setting they make will 739 * allow that much actual data to be received on that 740 * socket. 741 * 742 * Applications are unaware that "struct sk_buff" and 743 * other overheads allocate from the receive buffer 744 * during socket buffer allocation. 745 * 746 * And after considering the possible alternatives, 747 * returning the value we actually used in getsockopt 748 * is the most desirable behavior. 749 */ 750 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF); 751 break; 752 753 case SO_RCVBUFFORCE: 754 if (!capable(CAP_NET_ADMIN)) { 755 ret = -EPERM; 756 break; 757 } 758 goto set_rcvbuf; 759 760 case SO_KEEPALIVE: 761 if (sk->sk_prot->keepalive) 762 sk->sk_prot->keepalive(sk, valbool); 763 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 764 break; 765 766 case SO_OOBINLINE: 767 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 768 break; 769 770 case SO_NO_CHECK: 771 sk->sk_no_check_tx = valbool; 772 break; 773 774 case SO_PRIORITY: 775 if ((val >= 0 && val <= 6) || 776 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 777 sk->sk_priority = val; 778 else 779 ret = -EPERM; 780 break; 781 782 case SO_LINGER: 783 if (optlen < sizeof(ling)) { 784 ret = -EINVAL; /* 1003.1g */ 785 break; 786 } 787 if (copy_from_user(&ling, optval, sizeof(ling))) { 788 ret = -EFAULT; 789 break; 790 } 791 if (!ling.l_onoff) 792 sock_reset_flag(sk, SOCK_LINGER); 793 else { 794 #if (BITS_PER_LONG == 32) 795 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ) 796 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT; 797 else 798 #endif 799 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ; 800 sock_set_flag(sk, SOCK_LINGER); 801 } 802 break; 803 804 case SO_BSDCOMPAT: 805 sock_warn_obsolete_bsdism("setsockopt"); 806 break; 807 808 case SO_PASSCRED: 809 if (valbool) 810 set_bit(SOCK_PASSCRED, &sock->flags); 811 else 812 clear_bit(SOCK_PASSCRED, &sock->flags); 813 break; 814 815 case SO_TIMESTAMP: 816 case SO_TIMESTAMPNS: 817 if (valbool) { 818 if (optname == SO_TIMESTAMP) 819 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 820 else 821 sock_set_flag(sk, SOCK_RCVTSTAMPNS); 822 sock_set_flag(sk, SOCK_RCVTSTAMP); 823 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 824 } else { 825 sock_reset_flag(sk, SOCK_RCVTSTAMP); 826 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 827 } 828 break; 829 830 case SO_TIMESTAMPING: 831 if (val & ~SOF_TIMESTAMPING_MASK) { 832 ret = -EINVAL; 833 break; 834 } 835 836 if (val & SOF_TIMESTAMPING_OPT_ID && 837 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 838 if (sk->sk_protocol == IPPROTO_TCP && 839 sk->sk_type == SOCK_STREAM) { 840 if ((1 << sk->sk_state) & 841 (TCPF_CLOSE | TCPF_LISTEN)) { 842 ret = -EINVAL; 843 break; 844 } 845 sk->sk_tskey = tcp_sk(sk)->snd_una; 846 } else { 847 sk->sk_tskey = 0; 848 } 849 } 850 851 if (val & SOF_TIMESTAMPING_OPT_STATS && 852 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) { 853 ret = -EINVAL; 854 break; 855 } 856 857 sk->sk_tsflags = val; 858 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 859 sock_enable_timestamp(sk, 860 SOCK_TIMESTAMPING_RX_SOFTWARE); 861 else 862 sock_disable_timestamp(sk, 863 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 864 break; 865 866 case SO_RCVLOWAT: 867 if (val < 0) 868 val = INT_MAX; 869 if (sock->ops->set_rcvlowat) 870 ret = sock->ops->set_rcvlowat(sk, val); 871 else 872 sk->sk_rcvlowat = val ? : 1; 873 break; 874 875 case SO_RCVTIMEO: 876 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen); 877 break; 878 879 case SO_SNDTIMEO: 880 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen); 881 break; 882 883 case SO_ATTACH_FILTER: 884 ret = -EINVAL; 885 if (optlen == sizeof(struct sock_fprog)) { 886 struct sock_fprog fprog; 887 888 ret = -EFAULT; 889 if (copy_from_user(&fprog, optval, sizeof(fprog))) 890 break; 891 892 ret = sk_attach_filter(&fprog, sk); 893 } 894 break; 895 896 case SO_ATTACH_BPF: 897 ret = -EINVAL; 898 if (optlen == sizeof(u32)) { 899 u32 ufd; 900 901 ret = -EFAULT; 902 if (copy_from_user(&ufd, optval, sizeof(ufd))) 903 break; 904 905 ret = sk_attach_bpf(ufd, sk); 906 } 907 break; 908 909 case SO_ATTACH_REUSEPORT_CBPF: 910 ret = -EINVAL; 911 if (optlen == sizeof(struct sock_fprog)) { 912 struct sock_fprog fprog; 913 914 ret = -EFAULT; 915 if (copy_from_user(&fprog, optval, sizeof(fprog))) 916 break; 917 918 ret = sk_reuseport_attach_filter(&fprog, sk); 919 } 920 break; 921 922 case SO_ATTACH_REUSEPORT_EBPF: 923 ret = -EINVAL; 924 if (optlen == sizeof(u32)) { 925 u32 ufd; 926 927 ret = -EFAULT; 928 if (copy_from_user(&ufd, optval, sizeof(ufd))) 929 break; 930 931 ret = sk_reuseport_attach_bpf(ufd, sk); 932 } 933 break; 934 935 case SO_DETACH_FILTER: 936 ret = sk_detach_filter(sk); 937 break; 938 939 case SO_LOCK_FILTER: 940 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 941 ret = -EPERM; 942 else 943 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 944 break; 945 946 case SO_PASSSEC: 947 if (valbool) 948 set_bit(SOCK_PASSSEC, &sock->flags); 949 else 950 clear_bit(SOCK_PASSSEC, &sock->flags); 951 break; 952 case SO_MARK: 953 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 954 ret = -EPERM; 955 else 956 sk->sk_mark = val; 957 break; 958 959 case SO_RXQ_OVFL: 960 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 961 break; 962 963 case SO_WIFI_STATUS: 964 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 965 break; 966 967 case SO_PEEK_OFF: 968 if (sock->ops->set_peek_off) 969 ret = sock->ops->set_peek_off(sk, val); 970 else 971 ret = -EOPNOTSUPP; 972 break; 973 974 case SO_NOFCS: 975 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 976 break; 977 978 case SO_SELECT_ERR_QUEUE: 979 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 980 break; 981 982 #ifdef CONFIG_NET_RX_BUSY_POLL 983 case SO_BUSY_POLL: 984 /* allow unprivileged users to decrease the value */ 985 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN)) 986 ret = -EPERM; 987 else { 988 if (val < 0) 989 ret = -EINVAL; 990 else 991 sk->sk_ll_usec = val; 992 } 993 break; 994 #endif 995 996 case SO_MAX_PACING_RATE: 997 if (val != ~0U) 998 cmpxchg(&sk->sk_pacing_status, 999 SK_PACING_NONE, 1000 SK_PACING_NEEDED); 1001 sk->sk_max_pacing_rate = (val == ~0U) ? ~0UL : val; 1002 sk->sk_pacing_rate = min(sk->sk_pacing_rate, 1003 sk->sk_max_pacing_rate); 1004 break; 1005 1006 case SO_INCOMING_CPU: 1007 sk->sk_incoming_cpu = val; 1008 break; 1009 1010 case SO_CNX_ADVICE: 1011 if (val == 1) 1012 dst_negative_advice(sk); 1013 break; 1014 1015 case SO_ZEROCOPY: 1016 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1017 if (sk->sk_protocol != IPPROTO_TCP) 1018 ret = -ENOTSUPP; 1019 } else if (sk->sk_family != PF_RDS) { 1020 ret = -ENOTSUPP; 1021 } 1022 if (!ret) { 1023 if (val < 0 || val > 1) 1024 ret = -EINVAL; 1025 else 1026 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1027 } 1028 break; 1029 1030 case SO_TXTIME: 1031 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1032 ret = -EPERM; 1033 } else if (optlen != sizeof(struct sock_txtime)) { 1034 ret = -EINVAL; 1035 } else if (copy_from_user(&sk_txtime, optval, 1036 sizeof(struct sock_txtime))) { 1037 ret = -EFAULT; 1038 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1039 ret = -EINVAL; 1040 } else { 1041 sock_valbool_flag(sk, SOCK_TXTIME, true); 1042 sk->sk_clockid = sk_txtime.clockid; 1043 sk->sk_txtime_deadline_mode = 1044 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1045 sk->sk_txtime_report_errors = 1046 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1047 } 1048 break; 1049 1050 default: 1051 ret = -ENOPROTOOPT; 1052 break; 1053 } 1054 release_sock(sk); 1055 return ret; 1056 } 1057 EXPORT_SYMBOL(sock_setsockopt); 1058 1059 1060 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1061 struct ucred *ucred) 1062 { 1063 ucred->pid = pid_vnr(pid); 1064 ucred->uid = ucred->gid = -1; 1065 if (cred) { 1066 struct user_namespace *current_ns = current_user_ns(); 1067 1068 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1069 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1070 } 1071 } 1072 1073 static int groups_to_user(gid_t __user *dst, const struct group_info *src) 1074 { 1075 struct user_namespace *user_ns = current_user_ns(); 1076 int i; 1077 1078 for (i = 0; i < src->ngroups; i++) 1079 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i)) 1080 return -EFAULT; 1081 1082 return 0; 1083 } 1084 1085 int sock_getsockopt(struct socket *sock, int level, int optname, 1086 char __user *optval, int __user *optlen) 1087 { 1088 struct sock *sk = sock->sk; 1089 1090 union { 1091 int val; 1092 u64 val64; 1093 struct linger ling; 1094 struct timeval tm; 1095 struct sock_txtime txtime; 1096 } v; 1097 1098 int lv = sizeof(int); 1099 int len; 1100 1101 if (get_user(len, optlen)) 1102 return -EFAULT; 1103 if (len < 0) 1104 return -EINVAL; 1105 1106 memset(&v, 0, sizeof(v)); 1107 1108 switch (optname) { 1109 case SO_DEBUG: 1110 v.val = sock_flag(sk, SOCK_DBG); 1111 break; 1112 1113 case SO_DONTROUTE: 1114 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1115 break; 1116 1117 case SO_BROADCAST: 1118 v.val = sock_flag(sk, SOCK_BROADCAST); 1119 break; 1120 1121 case SO_SNDBUF: 1122 v.val = sk->sk_sndbuf; 1123 break; 1124 1125 case SO_RCVBUF: 1126 v.val = sk->sk_rcvbuf; 1127 break; 1128 1129 case SO_REUSEADDR: 1130 v.val = sk->sk_reuse; 1131 break; 1132 1133 case SO_REUSEPORT: 1134 v.val = sk->sk_reuseport; 1135 break; 1136 1137 case SO_KEEPALIVE: 1138 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1139 break; 1140 1141 case SO_TYPE: 1142 v.val = sk->sk_type; 1143 break; 1144 1145 case SO_PROTOCOL: 1146 v.val = sk->sk_protocol; 1147 break; 1148 1149 case SO_DOMAIN: 1150 v.val = sk->sk_family; 1151 break; 1152 1153 case SO_ERROR: 1154 v.val = -sock_error(sk); 1155 if (v.val == 0) 1156 v.val = xchg(&sk->sk_err_soft, 0); 1157 break; 1158 1159 case SO_OOBINLINE: 1160 v.val = sock_flag(sk, SOCK_URGINLINE); 1161 break; 1162 1163 case SO_NO_CHECK: 1164 v.val = sk->sk_no_check_tx; 1165 break; 1166 1167 case SO_PRIORITY: 1168 v.val = sk->sk_priority; 1169 break; 1170 1171 case SO_LINGER: 1172 lv = sizeof(v.ling); 1173 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1174 v.ling.l_linger = sk->sk_lingertime / HZ; 1175 break; 1176 1177 case SO_BSDCOMPAT: 1178 sock_warn_obsolete_bsdism("getsockopt"); 1179 break; 1180 1181 case SO_TIMESTAMP: 1182 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1183 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1184 break; 1185 1186 case SO_TIMESTAMPNS: 1187 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS); 1188 break; 1189 1190 case SO_TIMESTAMPING: 1191 v.val = sk->sk_tsflags; 1192 break; 1193 1194 case SO_RCVTIMEO: 1195 lv = sizeof(struct timeval); 1196 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) { 1197 v.tm.tv_sec = 0; 1198 v.tm.tv_usec = 0; 1199 } else { 1200 v.tm.tv_sec = sk->sk_rcvtimeo / HZ; 1201 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ; 1202 } 1203 break; 1204 1205 case SO_SNDTIMEO: 1206 lv = sizeof(struct timeval); 1207 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) { 1208 v.tm.tv_sec = 0; 1209 v.tm.tv_usec = 0; 1210 } else { 1211 v.tm.tv_sec = sk->sk_sndtimeo / HZ; 1212 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ; 1213 } 1214 break; 1215 1216 case SO_RCVLOWAT: 1217 v.val = sk->sk_rcvlowat; 1218 break; 1219 1220 case SO_SNDLOWAT: 1221 v.val = 1; 1222 break; 1223 1224 case SO_PASSCRED: 1225 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1226 break; 1227 1228 case SO_PEERCRED: 1229 { 1230 struct ucred peercred; 1231 if (len > sizeof(peercred)) 1232 len = sizeof(peercred); 1233 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1234 if (copy_to_user(optval, &peercred, len)) 1235 return -EFAULT; 1236 goto lenout; 1237 } 1238 1239 case SO_PEERGROUPS: 1240 { 1241 int ret, n; 1242 1243 if (!sk->sk_peer_cred) 1244 return -ENODATA; 1245 1246 n = sk->sk_peer_cred->group_info->ngroups; 1247 if (len < n * sizeof(gid_t)) { 1248 len = n * sizeof(gid_t); 1249 return put_user(len, optlen) ? -EFAULT : -ERANGE; 1250 } 1251 len = n * sizeof(gid_t); 1252 1253 ret = groups_to_user((gid_t __user *)optval, 1254 sk->sk_peer_cred->group_info); 1255 if (ret) 1256 return ret; 1257 goto lenout; 1258 } 1259 1260 case SO_PEERNAME: 1261 { 1262 char address[128]; 1263 1264 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2); 1265 if (lv < 0) 1266 return -ENOTCONN; 1267 if (lv < len) 1268 return -EINVAL; 1269 if (copy_to_user(optval, address, len)) 1270 return -EFAULT; 1271 goto lenout; 1272 } 1273 1274 /* Dubious BSD thing... Probably nobody even uses it, but 1275 * the UNIX standard wants it for whatever reason... -DaveM 1276 */ 1277 case SO_ACCEPTCONN: 1278 v.val = sk->sk_state == TCP_LISTEN; 1279 break; 1280 1281 case SO_PASSSEC: 1282 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1283 break; 1284 1285 case SO_PEERSEC: 1286 return security_socket_getpeersec_stream(sock, optval, optlen, len); 1287 1288 case SO_MARK: 1289 v.val = sk->sk_mark; 1290 break; 1291 1292 case SO_RXQ_OVFL: 1293 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1294 break; 1295 1296 case SO_WIFI_STATUS: 1297 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1298 break; 1299 1300 case SO_PEEK_OFF: 1301 if (!sock->ops->set_peek_off) 1302 return -EOPNOTSUPP; 1303 1304 v.val = sk->sk_peek_off; 1305 break; 1306 case SO_NOFCS: 1307 v.val = sock_flag(sk, SOCK_NOFCS); 1308 break; 1309 1310 case SO_BINDTODEVICE: 1311 return sock_getbindtodevice(sk, optval, optlen, len); 1312 1313 case SO_GET_FILTER: 1314 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len); 1315 if (len < 0) 1316 return len; 1317 1318 goto lenout; 1319 1320 case SO_LOCK_FILTER: 1321 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1322 break; 1323 1324 case SO_BPF_EXTENSIONS: 1325 v.val = bpf_tell_extensions(); 1326 break; 1327 1328 case SO_SELECT_ERR_QUEUE: 1329 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1330 break; 1331 1332 #ifdef CONFIG_NET_RX_BUSY_POLL 1333 case SO_BUSY_POLL: 1334 v.val = sk->sk_ll_usec; 1335 break; 1336 #endif 1337 1338 case SO_MAX_PACING_RATE: 1339 /* 32bit version */ 1340 v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U); 1341 break; 1342 1343 case SO_INCOMING_CPU: 1344 v.val = sk->sk_incoming_cpu; 1345 break; 1346 1347 case SO_MEMINFO: 1348 { 1349 u32 meminfo[SK_MEMINFO_VARS]; 1350 1351 if (get_user(len, optlen)) 1352 return -EFAULT; 1353 1354 sk_get_meminfo(sk, meminfo); 1355 1356 len = min_t(unsigned int, len, sizeof(meminfo)); 1357 if (copy_to_user(optval, &meminfo, len)) 1358 return -EFAULT; 1359 1360 goto lenout; 1361 } 1362 1363 #ifdef CONFIG_NET_RX_BUSY_POLL 1364 case SO_INCOMING_NAPI_ID: 1365 v.val = READ_ONCE(sk->sk_napi_id); 1366 1367 /* aggregate non-NAPI IDs down to 0 */ 1368 if (v.val < MIN_NAPI_ID) 1369 v.val = 0; 1370 1371 break; 1372 #endif 1373 1374 case SO_COOKIE: 1375 lv = sizeof(u64); 1376 if (len < lv) 1377 return -EINVAL; 1378 v.val64 = sock_gen_cookie(sk); 1379 break; 1380 1381 case SO_ZEROCOPY: 1382 v.val = sock_flag(sk, SOCK_ZEROCOPY); 1383 break; 1384 1385 case SO_TXTIME: 1386 lv = sizeof(v.txtime); 1387 v.txtime.clockid = sk->sk_clockid; 1388 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 1389 SOF_TXTIME_DEADLINE_MODE : 0; 1390 v.txtime.flags |= sk->sk_txtime_report_errors ? 1391 SOF_TXTIME_REPORT_ERRORS : 0; 1392 break; 1393 1394 default: 1395 /* We implement the SO_SNDLOWAT etc to not be settable 1396 * (1003.1g 7). 1397 */ 1398 return -ENOPROTOOPT; 1399 } 1400 1401 if (len > lv) 1402 len = lv; 1403 if (copy_to_user(optval, &v, len)) 1404 return -EFAULT; 1405 lenout: 1406 if (put_user(len, optlen)) 1407 return -EFAULT; 1408 return 0; 1409 } 1410 1411 /* 1412 * Initialize an sk_lock. 1413 * 1414 * (We also register the sk_lock with the lock validator.) 1415 */ 1416 static inline void sock_lock_init(struct sock *sk) 1417 { 1418 if (sk->sk_kern_sock) 1419 sock_lock_init_class_and_name( 1420 sk, 1421 af_family_kern_slock_key_strings[sk->sk_family], 1422 af_family_kern_slock_keys + sk->sk_family, 1423 af_family_kern_key_strings[sk->sk_family], 1424 af_family_kern_keys + sk->sk_family); 1425 else 1426 sock_lock_init_class_and_name( 1427 sk, 1428 af_family_slock_key_strings[sk->sk_family], 1429 af_family_slock_keys + sk->sk_family, 1430 af_family_key_strings[sk->sk_family], 1431 af_family_keys + sk->sk_family); 1432 } 1433 1434 /* 1435 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 1436 * even temporarly, because of RCU lookups. sk_node should also be left as is. 1437 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 1438 */ 1439 static void sock_copy(struct sock *nsk, const struct sock *osk) 1440 { 1441 #ifdef CONFIG_SECURITY_NETWORK 1442 void *sptr = nsk->sk_security; 1443 #endif 1444 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 1445 1446 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 1447 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 1448 1449 #ifdef CONFIG_SECURITY_NETWORK 1450 nsk->sk_security = sptr; 1451 security_sk_clone(osk, nsk); 1452 #endif 1453 } 1454 1455 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 1456 int family) 1457 { 1458 struct sock *sk; 1459 struct kmem_cache *slab; 1460 1461 slab = prot->slab; 1462 if (slab != NULL) { 1463 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 1464 if (!sk) 1465 return sk; 1466 if (priority & __GFP_ZERO) 1467 sk_prot_clear_nulls(sk, prot->obj_size); 1468 } else 1469 sk = kmalloc(prot->obj_size, priority); 1470 1471 if (sk != NULL) { 1472 if (security_sk_alloc(sk, family, priority)) 1473 goto out_free; 1474 1475 if (!try_module_get(prot->owner)) 1476 goto out_free_sec; 1477 sk_tx_queue_clear(sk); 1478 } 1479 1480 return sk; 1481 1482 out_free_sec: 1483 security_sk_free(sk); 1484 out_free: 1485 if (slab != NULL) 1486 kmem_cache_free(slab, sk); 1487 else 1488 kfree(sk); 1489 return NULL; 1490 } 1491 1492 static void sk_prot_free(struct proto *prot, struct sock *sk) 1493 { 1494 struct kmem_cache *slab; 1495 struct module *owner; 1496 1497 owner = prot->owner; 1498 slab = prot->slab; 1499 1500 cgroup_sk_free(&sk->sk_cgrp_data); 1501 mem_cgroup_sk_free(sk); 1502 security_sk_free(sk); 1503 if (slab != NULL) 1504 kmem_cache_free(slab, sk); 1505 else 1506 kfree(sk); 1507 module_put(owner); 1508 } 1509 1510 /** 1511 * sk_alloc - All socket objects are allocated here 1512 * @net: the applicable net namespace 1513 * @family: protocol family 1514 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1515 * @prot: struct proto associated with this new sock instance 1516 * @kern: is this to be a kernel socket? 1517 */ 1518 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 1519 struct proto *prot, int kern) 1520 { 1521 struct sock *sk; 1522 1523 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 1524 if (sk) { 1525 sk->sk_family = family; 1526 /* 1527 * See comment in struct sock definition to understand 1528 * why we need sk_prot_creator -acme 1529 */ 1530 sk->sk_prot = sk->sk_prot_creator = prot; 1531 sk->sk_kern_sock = kern; 1532 sock_lock_init(sk); 1533 sk->sk_net_refcnt = kern ? 0 : 1; 1534 if (likely(sk->sk_net_refcnt)) { 1535 get_net(net); 1536 sock_inuse_add(net, 1); 1537 } 1538 1539 sock_net_set(sk, net); 1540 refcount_set(&sk->sk_wmem_alloc, 1); 1541 1542 mem_cgroup_sk_alloc(sk); 1543 cgroup_sk_alloc(&sk->sk_cgrp_data); 1544 sock_update_classid(&sk->sk_cgrp_data); 1545 sock_update_netprioidx(&sk->sk_cgrp_data); 1546 } 1547 1548 return sk; 1549 } 1550 EXPORT_SYMBOL(sk_alloc); 1551 1552 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 1553 * grace period. This is the case for UDP sockets and TCP listeners. 1554 */ 1555 static void __sk_destruct(struct rcu_head *head) 1556 { 1557 struct sock *sk = container_of(head, struct sock, sk_rcu); 1558 struct sk_filter *filter; 1559 1560 if (sk->sk_destruct) 1561 sk->sk_destruct(sk); 1562 1563 filter = rcu_dereference_check(sk->sk_filter, 1564 refcount_read(&sk->sk_wmem_alloc) == 0); 1565 if (filter) { 1566 sk_filter_uncharge(sk, filter); 1567 RCU_INIT_POINTER(sk->sk_filter, NULL); 1568 } 1569 if (rcu_access_pointer(sk->sk_reuseport_cb)) 1570 reuseport_detach_sock(sk); 1571 1572 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 1573 1574 if (atomic_read(&sk->sk_omem_alloc)) 1575 pr_debug("%s: optmem leakage (%d bytes) detected\n", 1576 __func__, atomic_read(&sk->sk_omem_alloc)); 1577 1578 if (sk->sk_frag.page) { 1579 put_page(sk->sk_frag.page); 1580 sk->sk_frag.page = NULL; 1581 } 1582 1583 if (sk->sk_peer_cred) 1584 put_cred(sk->sk_peer_cred); 1585 put_pid(sk->sk_peer_pid); 1586 if (likely(sk->sk_net_refcnt)) 1587 put_net(sock_net(sk)); 1588 sk_prot_free(sk->sk_prot_creator, sk); 1589 } 1590 1591 void sk_destruct(struct sock *sk) 1592 { 1593 if (sock_flag(sk, SOCK_RCU_FREE)) 1594 call_rcu(&sk->sk_rcu, __sk_destruct); 1595 else 1596 __sk_destruct(&sk->sk_rcu); 1597 } 1598 1599 static void __sk_free(struct sock *sk) 1600 { 1601 if (likely(sk->sk_net_refcnt)) 1602 sock_inuse_add(sock_net(sk), -1); 1603 1604 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 1605 sock_diag_broadcast_destroy(sk); 1606 else 1607 sk_destruct(sk); 1608 } 1609 1610 void sk_free(struct sock *sk) 1611 { 1612 /* 1613 * We subtract one from sk_wmem_alloc and can know if 1614 * some packets are still in some tx queue. 1615 * If not null, sock_wfree() will call __sk_free(sk) later 1616 */ 1617 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 1618 __sk_free(sk); 1619 } 1620 EXPORT_SYMBOL(sk_free); 1621 1622 static void sk_init_common(struct sock *sk) 1623 { 1624 skb_queue_head_init(&sk->sk_receive_queue); 1625 skb_queue_head_init(&sk->sk_write_queue); 1626 skb_queue_head_init(&sk->sk_error_queue); 1627 1628 rwlock_init(&sk->sk_callback_lock); 1629 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 1630 af_rlock_keys + sk->sk_family, 1631 af_family_rlock_key_strings[sk->sk_family]); 1632 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 1633 af_wlock_keys + sk->sk_family, 1634 af_family_wlock_key_strings[sk->sk_family]); 1635 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 1636 af_elock_keys + sk->sk_family, 1637 af_family_elock_key_strings[sk->sk_family]); 1638 lockdep_set_class_and_name(&sk->sk_callback_lock, 1639 af_callback_keys + sk->sk_family, 1640 af_family_clock_key_strings[sk->sk_family]); 1641 } 1642 1643 /** 1644 * sk_clone_lock - clone a socket, and lock its clone 1645 * @sk: the socket to clone 1646 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1647 * 1648 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1649 */ 1650 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 1651 { 1652 struct sock *newsk; 1653 bool is_charged = true; 1654 1655 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family); 1656 if (newsk != NULL) { 1657 struct sk_filter *filter; 1658 1659 sock_copy(newsk, sk); 1660 1661 newsk->sk_prot_creator = sk->sk_prot; 1662 1663 /* SANITY */ 1664 if (likely(newsk->sk_net_refcnt)) 1665 get_net(sock_net(newsk)); 1666 sk_node_init(&newsk->sk_node); 1667 sock_lock_init(newsk); 1668 bh_lock_sock(newsk); 1669 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 1670 newsk->sk_backlog.len = 0; 1671 1672 atomic_set(&newsk->sk_rmem_alloc, 0); 1673 /* 1674 * sk_wmem_alloc set to one (see sk_free() and sock_wfree()) 1675 */ 1676 refcount_set(&newsk->sk_wmem_alloc, 1); 1677 atomic_set(&newsk->sk_omem_alloc, 0); 1678 sk_init_common(newsk); 1679 1680 newsk->sk_dst_cache = NULL; 1681 newsk->sk_dst_pending_confirm = 0; 1682 newsk->sk_wmem_queued = 0; 1683 newsk->sk_forward_alloc = 0; 1684 atomic_set(&newsk->sk_drops, 0); 1685 newsk->sk_send_head = NULL; 1686 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 1687 atomic_set(&newsk->sk_zckey, 0); 1688 1689 sock_reset_flag(newsk, SOCK_DONE); 1690 mem_cgroup_sk_alloc(newsk); 1691 cgroup_sk_alloc(&newsk->sk_cgrp_data); 1692 1693 rcu_read_lock(); 1694 filter = rcu_dereference(sk->sk_filter); 1695 if (filter != NULL) 1696 /* though it's an empty new sock, the charging may fail 1697 * if sysctl_optmem_max was changed between creation of 1698 * original socket and cloning 1699 */ 1700 is_charged = sk_filter_charge(newsk, filter); 1701 RCU_INIT_POINTER(newsk->sk_filter, filter); 1702 rcu_read_unlock(); 1703 1704 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 1705 /* We need to make sure that we don't uncharge the new 1706 * socket if we couldn't charge it in the first place 1707 * as otherwise we uncharge the parent's filter. 1708 */ 1709 if (!is_charged) 1710 RCU_INIT_POINTER(newsk->sk_filter, NULL); 1711 sk_free_unlock_clone(newsk); 1712 newsk = NULL; 1713 goto out; 1714 } 1715 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 1716 1717 newsk->sk_err = 0; 1718 newsk->sk_err_soft = 0; 1719 newsk->sk_priority = 0; 1720 newsk->sk_incoming_cpu = raw_smp_processor_id(); 1721 atomic64_set(&newsk->sk_cookie, 0); 1722 if (likely(newsk->sk_net_refcnt)) 1723 sock_inuse_add(sock_net(newsk), 1); 1724 1725 /* 1726 * Before updating sk_refcnt, we must commit prior changes to memory 1727 * (Documentation/RCU/rculist_nulls.txt for details) 1728 */ 1729 smp_wmb(); 1730 refcount_set(&newsk->sk_refcnt, 2); 1731 1732 /* 1733 * Increment the counter in the same struct proto as the master 1734 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that 1735 * is the same as sk->sk_prot->socks, as this field was copied 1736 * with memcpy). 1737 * 1738 * This _changes_ the previous behaviour, where 1739 * tcp_create_openreq_child always was incrementing the 1740 * equivalent to tcp_prot->socks (inet_sock_nr), so this have 1741 * to be taken into account in all callers. -acme 1742 */ 1743 sk_refcnt_debug_inc(newsk); 1744 sk_set_socket(newsk, NULL); 1745 newsk->sk_wq = NULL; 1746 1747 if (newsk->sk_prot->sockets_allocated) 1748 sk_sockets_allocated_inc(newsk); 1749 1750 if (sock_needs_netstamp(sk) && 1751 newsk->sk_flags & SK_FLAGS_TIMESTAMP) 1752 net_enable_timestamp(); 1753 } 1754 out: 1755 return newsk; 1756 } 1757 EXPORT_SYMBOL_GPL(sk_clone_lock); 1758 1759 void sk_free_unlock_clone(struct sock *sk) 1760 { 1761 /* It is still raw copy of parent, so invalidate 1762 * destructor and make plain sk_free() */ 1763 sk->sk_destruct = NULL; 1764 bh_unlock_sock(sk); 1765 sk_free(sk); 1766 } 1767 EXPORT_SYMBOL_GPL(sk_free_unlock_clone); 1768 1769 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 1770 { 1771 u32 max_segs = 1; 1772 1773 sk_dst_set(sk, dst); 1774 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps; 1775 if (sk->sk_route_caps & NETIF_F_GSO) 1776 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 1777 sk->sk_route_caps &= ~sk->sk_route_nocaps; 1778 if (sk_can_gso(sk)) { 1779 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 1780 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 1781 } else { 1782 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 1783 sk->sk_gso_max_size = dst->dev->gso_max_size; 1784 max_segs = max_t(u32, dst->dev->gso_max_segs, 1); 1785 } 1786 } 1787 sk->sk_gso_max_segs = max_segs; 1788 } 1789 EXPORT_SYMBOL_GPL(sk_setup_caps); 1790 1791 /* 1792 * Simple resource managers for sockets. 1793 */ 1794 1795 1796 /* 1797 * Write buffer destructor automatically called from kfree_skb. 1798 */ 1799 void sock_wfree(struct sk_buff *skb) 1800 { 1801 struct sock *sk = skb->sk; 1802 unsigned int len = skb->truesize; 1803 1804 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 1805 /* 1806 * Keep a reference on sk_wmem_alloc, this will be released 1807 * after sk_write_space() call 1808 */ 1809 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 1810 sk->sk_write_space(sk); 1811 len = 1; 1812 } 1813 /* 1814 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 1815 * could not do because of in-flight packets 1816 */ 1817 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 1818 __sk_free(sk); 1819 } 1820 EXPORT_SYMBOL(sock_wfree); 1821 1822 /* This variant of sock_wfree() is used by TCP, 1823 * since it sets SOCK_USE_WRITE_QUEUE. 1824 */ 1825 void __sock_wfree(struct sk_buff *skb) 1826 { 1827 struct sock *sk = skb->sk; 1828 1829 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 1830 __sk_free(sk); 1831 } 1832 1833 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 1834 { 1835 skb_orphan(skb); 1836 skb->sk = sk; 1837 #ifdef CONFIG_INET 1838 if (unlikely(!sk_fullsock(sk))) { 1839 skb->destructor = sock_edemux; 1840 sock_hold(sk); 1841 return; 1842 } 1843 #endif 1844 skb->destructor = sock_wfree; 1845 skb_set_hash_from_sk(skb, sk); 1846 /* 1847 * We used to take a refcount on sk, but following operation 1848 * is enough to guarantee sk_free() wont free this sock until 1849 * all in-flight packets are completed 1850 */ 1851 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 1852 } 1853 EXPORT_SYMBOL(skb_set_owner_w); 1854 1855 /* This helper is used by netem, as it can hold packets in its 1856 * delay queue. We want to allow the owner socket to send more 1857 * packets, as if they were already TX completed by a typical driver. 1858 * But we also want to keep skb->sk set because some packet schedulers 1859 * rely on it (sch_fq for example). 1860 */ 1861 void skb_orphan_partial(struct sk_buff *skb) 1862 { 1863 if (skb_is_tcp_pure_ack(skb)) 1864 return; 1865 1866 if (skb->destructor == sock_wfree 1867 #ifdef CONFIG_INET 1868 || skb->destructor == tcp_wfree 1869 #endif 1870 ) { 1871 struct sock *sk = skb->sk; 1872 1873 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 1874 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)); 1875 skb->destructor = sock_efree; 1876 } 1877 } else { 1878 skb_orphan(skb); 1879 } 1880 } 1881 EXPORT_SYMBOL(skb_orphan_partial); 1882 1883 /* 1884 * Read buffer destructor automatically called from kfree_skb. 1885 */ 1886 void sock_rfree(struct sk_buff *skb) 1887 { 1888 struct sock *sk = skb->sk; 1889 unsigned int len = skb->truesize; 1890 1891 atomic_sub(len, &sk->sk_rmem_alloc); 1892 sk_mem_uncharge(sk, len); 1893 } 1894 EXPORT_SYMBOL(sock_rfree); 1895 1896 /* 1897 * Buffer destructor for skbs that are not used directly in read or write 1898 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 1899 */ 1900 void sock_efree(struct sk_buff *skb) 1901 { 1902 sock_put(skb->sk); 1903 } 1904 EXPORT_SYMBOL(sock_efree); 1905 1906 kuid_t sock_i_uid(struct sock *sk) 1907 { 1908 kuid_t uid; 1909 1910 read_lock_bh(&sk->sk_callback_lock); 1911 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 1912 read_unlock_bh(&sk->sk_callback_lock); 1913 return uid; 1914 } 1915 EXPORT_SYMBOL(sock_i_uid); 1916 1917 unsigned long sock_i_ino(struct sock *sk) 1918 { 1919 unsigned long ino; 1920 1921 read_lock_bh(&sk->sk_callback_lock); 1922 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 1923 read_unlock_bh(&sk->sk_callback_lock); 1924 return ino; 1925 } 1926 EXPORT_SYMBOL(sock_i_ino); 1927 1928 /* 1929 * Allocate a skb from the socket's send buffer. 1930 */ 1931 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 1932 gfp_t priority) 1933 { 1934 if (force || refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) { 1935 struct sk_buff *skb = alloc_skb(size, priority); 1936 if (skb) { 1937 skb_set_owner_w(skb, sk); 1938 return skb; 1939 } 1940 } 1941 return NULL; 1942 } 1943 EXPORT_SYMBOL(sock_wmalloc); 1944 1945 static void sock_ofree(struct sk_buff *skb) 1946 { 1947 struct sock *sk = skb->sk; 1948 1949 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 1950 } 1951 1952 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 1953 gfp_t priority) 1954 { 1955 struct sk_buff *skb; 1956 1957 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 1958 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 1959 sysctl_optmem_max) 1960 return NULL; 1961 1962 skb = alloc_skb(size, priority); 1963 if (!skb) 1964 return NULL; 1965 1966 atomic_add(skb->truesize, &sk->sk_omem_alloc); 1967 skb->sk = sk; 1968 skb->destructor = sock_ofree; 1969 return skb; 1970 } 1971 1972 /* 1973 * Allocate a memory block from the socket's option memory buffer. 1974 */ 1975 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 1976 { 1977 if ((unsigned int)size <= sysctl_optmem_max && 1978 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { 1979 void *mem; 1980 /* First do the add, to avoid the race if kmalloc 1981 * might sleep. 1982 */ 1983 atomic_add(size, &sk->sk_omem_alloc); 1984 mem = kmalloc(size, priority); 1985 if (mem) 1986 return mem; 1987 atomic_sub(size, &sk->sk_omem_alloc); 1988 } 1989 return NULL; 1990 } 1991 EXPORT_SYMBOL(sock_kmalloc); 1992 1993 /* Free an option memory block. Note, we actually want the inline 1994 * here as this allows gcc to detect the nullify and fold away the 1995 * condition entirely. 1996 */ 1997 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 1998 const bool nullify) 1999 { 2000 if (WARN_ON_ONCE(!mem)) 2001 return; 2002 if (nullify) 2003 kzfree(mem); 2004 else 2005 kfree(mem); 2006 atomic_sub(size, &sk->sk_omem_alloc); 2007 } 2008 2009 void sock_kfree_s(struct sock *sk, void *mem, int size) 2010 { 2011 __sock_kfree_s(sk, mem, size, false); 2012 } 2013 EXPORT_SYMBOL(sock_kfree_s); 2014 2015 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2016 { 2017 __sock_kfree_s(sk, mem, size, true); 2018 } 2019 EXPORT_SYMBOL(sock_kzfree_s); 2020 2021 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2022 I think, these locks should be removed for datagram sockets. 2023 */ 2024 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2025 { 2026 DEFINE_WAIT(wait); 2027 2028 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2029 for (;;) { 2030 if (!timeo) 2031 break; 2032 if (signal_pending(current)) 2033 break; 2034 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2035 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2036 if (refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) 2037 break; 2038 if (sk->sk_shutdown & SEND_SHUTDOWN) 2039 break; 2040 if (sk->sk_err) 2041 break; 2042 timeo = schedule_timeout(timeo); 2043 } 2044 finish_wait(sk_sleep(sk), &wait); 2045 return timeo; 2046 } 2047 2048 2049 /* 2050 * Generic send/receive buffer handlers 2051 */ 2052 2053 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2054 unsigned long data_len, int noblock, 2055 int *errcode, int max_page_order) 2056 { 2057 struct sk_buff *skb; 2058 long timeo; 2059 int err; 2060 2061 timeo = sock_sndtimeo(sk, noblock); 2062 for (;;) { 2063 err = sock_error(sk); 2064 if (err != 0) 2065 goto failure; 2066 2067 err = -EPIPE; 2068 if (sk->sk_shutdown & SEND_SHUTDOWN) 2069 goto failure; 2070 2071 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf) 2072 break; 2073 2074 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2075 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2076 err = -EAGAIN; 2077 if (!timeo) 2078 goto failure; 2079 if (signal_pending(current)) 2080 goto interrupted; 2081 timeo = sock_wait_for_wmem(sk, timeo); 2082 } 2083 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 2084 errcode, sk->sk_allocation); 2085 if (skb) 2086 skb_set_owner_w(skb, sk); 2087 return skb; 2088 2089 interrupted: 2090 err = sock_intr_errno(timeo); 2091 failure: 2092 *errcode = err; 2093 return NULL; 2094 } 2095 EXPORT_SYMBOL(sock_alloc_send_pskb); 2096 2097 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size, 2098 int noblock, int *errcode) 2099 { 2100 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0); 2101 } 2102 EXPORT_SYMBOL(sock_alloc_send_skb); 2103 2104 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, 2105 struct sockcm_cookie *sockc) 2106 { 2107 u32 tsflags; 2108 2109 switch (cmsg->cmsg_type) { 2110 case SO_MARK: 2111 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 2112 return -EPERM; 2113 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2114 return -EINVAL; 2115 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 2116 break; 2117 case SO_TIMESTAMPING: 2118 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2119 return -EINVAL; 2120 2121 tsflags = *(u32 *)CMSG_DATA(cmsg); 2122 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 2123 return -EINVAL; 2124 2125 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 2126 sockc->tsflags |= tsflags; 2127 break; 2128 case SCM_TXTIME: 2129 if (!sock_flag(sk, SOCK_TXTIME)) 2130 return -EINVAL; 2131 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 2132 return -EINVAL; 2133 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); 2134 break; 2135 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 2136 case SCM_RIGHTS: 2137 case SCM_CREDENTIALS: 2138 break; 2139 default: 2140 return -EINVAL; 2141 } 2142 return 0; 2143 } 2144 EXPORT_SYMBOL(__sock_cmsg_send); 2145 2146 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 2147 struct sockcm_cookie *sockc) 2148 { 2149 struct cmsghdr *cmsg; 2150 int ret; 2151 2152 for_each_cmsghdr(cmsg, msg) { 2153 if (!CMSG_OK(msg, cmsg)) 2154 return -EINVAL; 2155 if (cmsg->cmsg_level != SOL_SOCKET) 2156 continue; 2157 ret = __sock_cmsg_send(sk, msg, cmsg, sockc); 2158 if (ret) 2159 return ret; 2160 } 2161 return 0; 2162 } 2163 EXPORT_SYMBOL(sock_cmsg_send); 2164 2165 static void sk_enter_memory_pressure(struct sock *sk) 2166 { 2167 if (!sk->sk_prot->enter_memory_pressure) 2168 return; 2169 2170 sk->sk_prot->enter_memory_pressure(sk); 2171 } 2172 2173 static void sk_leave_memory_pressure(struct sock *sk) 2174 { 2175 if (sk->sk_prot->leave_memory_pressure) { 2176 sk->sk_prot->leave_memory_pressure(sk); 2177 } else { 2178 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 2179 2180 if (memory_pressure && *memory_pressure) 2181 *memory_pressure = 0; 2182 } 2183 } 2184 2185 /* On 32bit arches, an skb frag is limited to 2^15 */ 2186 #define SKB_FRAG_PAGE_ORDER get_order(32768) 2187 2188 /** 2189 * skb_page_frag_refill - check that a page_frag contains enough room 2190 * @sz: minimum size of the fragment we want to get 2191 * @pfrag: pointer to page_frag 2192 * @gfp: priority for memory allocation 2193 * 2194 * Note: While this allocator tries to use high order pages, there is 2195 * no guarantee that allocations succeed. Therefore, @sz MUST be 2196 * less or equal than PAGE_SIZE. 2197 */ 2198 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 2199 { 2200 if (pfrag->page) { 2201 if (page_ref_count(pfrag->page) == 1) { 2202 pfrag->offset = 0; 2203 return true; 2204 } 2205 if (pfrag->offset + sz <= pfrag->size) 2206 return true; 2207 put_page(pfrag->page); 2208 } 2209 2210 pfrag->offset = 0; 2211 if (SKB_FRAG_PAGE_ORDER) { 2212 /* Avoid direct reclaim but allow kswapd to wake */ 2213 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 2214 __GFP_COMP | __GFP_NOWARN | 2215 __GFP_NORETRY, 2216 SKB_FRAG_PAGE_ORDER); 2217 if (likely(pfrag->page)) { 2218 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 2219 return true; 2220 } 2221 } 2222 pfrag->page = alloc_page(gfp); 2223 if (likely(pfrag->page)) { 2224 pfrag->size = PAGE_SIZE; 2225 return true; 2226 } 2227 return false; 2228 } 2229 EXPORT_SYMBOL(skb_page_frag_refill); 2230 2231 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2232 { 2233 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2234 return true; 2235 2236 sk_enter_memory_pressure(sk); 2237 sk_stream_moderate_sndbuf(sk); 2238 return false; 2239 } 2240 EXPORT_SYMBOL(sk_page_frag_refill); 2241 2242 static void __lock_sock(struct sock *sk) 2243 __releases(&sk->sk_lock.slock) 2244 __acquires(&sk->sk_lock.slock) 2245 { 2246 DEFINE_WAIT(wait); 2247 2248 for (;;) { 2249 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2250 TASK_UNINTERRUPTIBLE); 2251 spin_unlock_bh(&sk->sk_lock.slock); 2252 schedule(); 2253 spin_lock_bh(&sk->sk_lock.slock); 2254 if (!sock_owned_by_user(sk)) 2255 break; 2256 } 2257 finish_wait(&sk->sk_lock.wq, &wait); 2258 } 2259 2260 void __release_sock(struct sock *sk) 2261 __releases(&sk->sk_lock.slock) 2262 __acquires(&sk->sk_lock.slock) 2263 { 2264 struct sk_buff *skb, *next; 2265 2266 while ((skb = sk->sk_backlog.head) != NULL) { 2267 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2268 2269 spin_unlock_bh(&sk->sk_lock.slock); 2270 2271 do { 2272 next = skb->next; 2273 prefetch(next); 2274 WARN_ON_ONCE(skb_dst_is_noref(skb)); 2275 skb_mark_not_on_list(skb); 2276 sk_backlog_rcv(sk, skb); 2277 2278 cond_resched(); 2279 2280 skb = next; 2281 } while (skb != NULL); 2282 2283 spin_lock_bh(&sk->sk_lock.slock); 2284 } 2285 2286 /* 2287 * Doing the zeroing here guarantee we can not loop forever 2288 * while a wild producer attempts to flood us. 2289 */ 2290 sk->sk_backlog.len = 0; 2291 } 2292 2293 void __sk_flush_backlog(struct sock *sk) 2294 { 2295 spin_lock_bh(&sk->sk_lock.slock); 2296 __release_sock(sk); 2297 spin_unlock_bh(&sk->sk_lock.slock); 2298 } 2299 2300 /** 2301 * sk_wait_data - wait for data to arrive at sk_receive_queue 2302 * @sk: sock to wait on 2303 * @timeo: for how long 2304 * @skb: last skb seen on sk_receive_queue 2305 * 2306 * Now socket state including sk->sk_err is changed only under lock, 2307 * hence we may omit checks after joining wait queue. 2308 * We check receive queue before schedule() only as optimization; 2309 * it is very likely that release_sock() added new data. 2310 */ 2311 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 2312 { 2313 DEFINE_WAIT_FUNC(wait, woken_wake_function); 2314 int rc; 2315 2316 add_wait_queue(sk_sleep(sk), &wait); 2317 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2318 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 2319 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2320 remove_wait_queue(sk_sleep(sk), &wait); 2321 return rc; 2322 } 2323 EXPORT_SYMBOL(sk_wait_data); 2324 2325 /** 2326 * __sk_mem_raise_allocated - increase memory_allocated 2327 * @sk: socket 2328 * @size: memory size to allocate 2329 * @amt: pages to allocate 2330 * @kind: allocation type 2331 * 2332 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc 2333 */ 2334 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 2335 { 2336 struct proto *prot = sk->sk_prot; 2337 long allocated = sk_memory_allocated_add(sk, amt); 2338 bool charged = true; 2339 2340 if (mem_cgroup_sockets_enabled && sk->sk_memcg && 2341 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt))) 2342 goto suppress_allocation; 2343 2344 /* Under limit. */ 2345 if (allocated <= sk_prot_mem_limits(sk, 0)) { 2346 sk_leave_memory_pressure(sk); 2347 return 1; 2348 } 2349 2350 /* Under pressure. */ 2351 if (allocated > sk_prot_mem_limits(sk, 1)) 2352 sk_enter_memory_pressure(sk); 2353 2354 /* Over hard limit. */ 2355 if (allocated > sk_prot_mem_limits(sk, 2)) 2356 goto suppress_allocation; 2357 2358 /* guarantee minimum buffer size under pressure */ 2359 if (kind == SK_MEM_RECV) { 2360 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 2361 return 1; 2362 2363 } else { /* SK_MEM_SEND */ 2364 int wmem0 = sk_get_wmem0(sk, prot); 2365 2366 if (sk->sk_type == SOCK_STREAM) { 2367 if (sk->sk_wmem_queued < wmem0) 2368 return 1; 2369 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 2370 return 1; 2371 } 2372 } 2373 2374 if (sk_has_memory_pressure(sk)) { 2375 int alloc; 2376 2377 if (!sk_under_memory_pressure(sk)) 2378 return 1; 2379 alloc = sk_sockets_allocated_read_positive(sk); 2380 if (sk_prot_mem_limits(sk, 2) > alloc * 2381 sk_mem_pages(sk->sk_wmem_queued + 2382 atomic_read(&sk->sk_rmem_alloc) + 2383 sk->sk_forward_alloc)) 2384 return 1; 2385 } 2386 2387 suppress_allocation: 2388 2389 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 2390 sk_stream_moderate_sndbuf(sk); 2391 2392 /* Fail only if socket is _under_ its sndbuf. 2393 * In this case we cannot block, so that we have to fail. 2394 */ 2395 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) 2396 return 1; 2397 } 2398 2399 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged)) 2400 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 2401 2402 sk_memory_allocated_sub(sk, amt); 2403 2404 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2405 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt); 2406 2407 return 0; 2408 } 2409 EXPORT_SYMBOL(__sk_mem_raise_allocated); 2410 2411 /** 2412 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 2413 * @sk: socket 2414 * @size: memory size to allocate 2415 * @kind: allocation type 2416 * 2417 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 2418 * rmem allocation. This function assumes that protocols which have 2419 * memory_pressure use sk_wmem_queued as write buffer accounting. 2420 */ 2421 int __sk_mem_schedule(struct sock *sk, int size, int kind) 2422 { 2423 int ret, amt = sk_mem_pages(size); 2424 2425 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT; 2426 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 2427 if (!ret) 2428 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT; 2429 return ret; 2430 } 2431 EXPORT_SYMBOL(__sk_mem_schedule); 2432 2433 /** 2434 * __sk_mem_reduce_allocated - reclaim memory_allocated 2435 * @sk: socket 2436 * @amount: number of quanta 2437 * 2438 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 2439 */ 2440 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 2441 { 2442 sk_memory_allocated_sub(sk, amount); 2443 2444 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2445 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 2446 2447 if (sk_under_memory_pressure(sk) && 2448 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 2449 sk_leave_memory_pressure(sk); 2450 } 2451 EXPORT_SYMBOL(__sk_mem_reduce_allocated); 2452 2453 /** 2454 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 2455 * @sk: socket 2456 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple) 2457 */ 2458 void __sk_mem_reclaim(struct sock *sk, int amount) 2459 { 2460 amount >>= SK_MEM_QUANTUM_SHIFT; 2461 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT; 2462 __sk_mem_reduce_allocated(sk, amount); 2463 } 2464 EXPORT_SYMBOL(__sk_mem_reclaim); 2465 2466 int sk_set_peek_off(struct sock *sk, int val) 2467 { 2468 sk->sk_peek_off = val; 2469 return 0; 2470 } 2471 EXPORT_SYMBOL_GPL(sk_set_peek_off); 2472 2473 /* 2474 * Set of default routines for initialising struct proto_ops when 2475 * the protocol does not support a particular function. In certain 2476 * cases where it makes no sense for a protocol to have a "do nothing" 2477 * function, some default processing is provided. 2478 */ 2479 2480 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 2481 { 2482 return -EOPNOTSUPP; 2483 } 2484 EXPORT_SYMBOL(sock_no_bind); 2485 2486 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 2487 int len, int flags) 2488 { 2489 return -EOPNOTSUPP; 2490 } 2491 EXPORT_SYMBOL(sock_no_connect); 2492 2493 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 2494 { 2495 return -EOPNOTSUPP; 2496 } 2497 EXPORT_SYMBOL(sock_no_socketpair); 2498 2499 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, 2500 bool kern) 2501 { 2502 return -EOPNOTSUPP; 2503 } 2504 EXPORT_SYMBOL(sock_no_accept); 2505 2506 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 2507 int peer) 2508 { 2509 return -EOPNOTSUPP; 2510 } 2511 EXPORT_SYMBOL(sock_no_getname); 2512 2513 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 2514 { 2515 return -EOPNOTSUPP; 2516 } 2517 EXPORT_SYMBOL(sock_no_ioctl); 2518 2519 int sock_no_listen(struct socket *sock, int backlog) 2520 { 2521 return -EOPNOTSUPP; 2522 } 2523 EXPORT_SYMBOL(sock_no_listen); 2524 2525 int sock_no_shutdown(struct socket *sock, int how) 2526 { 2527 return -EOPNOTSUPP; 2528 } 2529 EXPORT_SYMBOL(sock_no_shutdown); 2530 2531 int sock_no_setsockopt(struct socket *sock, int level, int optname, 2532 char __user *optval, unsigned int optlen) 2533 { 2534 return -EOPNOTSUPP; 2535 } 2536 EXPORT_SYMBOL(sock_no_setsockopt); 2537 2538 int sock_no_getsockopt(struct socket *sock, int level, int optname, 2539 char __user *optval, int __user *optlen) 2540 { 2541 return -EOPNOTSUPP; 2542 } 2543 EXPORT_SYMBOL(sock_no_getsockopt); 2544 2545 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 2546 { 2547 return -EOPNOTSUPP; 2548 } 2549 EXPORT_SYMBOL(sock_no_sendmsg); 2550 2551 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 2552 { 2553 return -EOPNOTSUPP; 2554 } 2555 EXPORT_SYMBOL(sock_no_sendmsg_locked); 2556 2557 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 2558 int flags) 2559 { 2560 return -EOPNOTSUPP; 2561 } 2562 EXPORT_SYMBOL(sock_no_recvmsg); 2563 2564 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 2565 { 2566 /* Mirror missing mmap method error code */ 2567 return -ENODEV; 2568 } 2569 EXPORT_SYMBOL(sock_no_mmap); 2570 2571 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) 2572 { 2573 ssize_t res; 2574 struct msghdr msg = {.msg_flags = flags}; 2575 struct kvec iov; 2576 char *kaddr = kmap(page); 2577 iov.iov_base = kaddr + offset; 2578 iov.iov_len = size; 2579 res = kernel_sendmsg(sock, &msg, &iov, 1, size); 2580 kunmap(page); 2581 return res; 2582 } 2583 EXPORT_SYMBOL(sock_no_sendpage); 2584 2585 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page, 2586 int offset, size_t size, int flags) 2587 { 2588 ssize_t res; 2589 struct msghdr msg = {.msg_flags = flags}; 2590 struct kvec iov; 2591 char *kaddr = kmap(page); 2592 2593 iov.iov_base = kaddr + offset; 2594 iov.iov_len = size; 2595 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size); 2596 kunmap(page); 2597 return res; 2598 } 2599 EXPORT_SYMBOL(sock_no_sendpage_locked); 2600 2601 /* 2602 * Default Socket Callbacks 2603 */ 2604 2605 static void sock_def_wakeup(struct sock *sk) 2606 { 2607 struct socket_wq *wq; 2608 2609 rcu_read_lock(); 2610 wq = rcu_dereference(sk->sk_wq); 2611 if (skwq_has_sleeper(wq)) 2612 wake_up_interruptible_all(&wq->wait); 2613 rcu_read_unlock(); 2614 } 2615 2616 static void sock_def_error_report(struct sock *sk) 2617 { 2618 struct socket_wq *wq; 2619 2620 rcu_read_lock(); 2621 wq = rcu_dereference(sk->sk_wq); 2622 if (skwq_has_sleeper(wq)) 2623 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 2624 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 2625 rcu_read_unlock(); 2626 } 2627 2628 static void sock_def_readable(struct sock *sk) 2629 { 2630 struct socket_wq *wq; 2631 2632 rcu_read_lock(); 2633 wq = rcu_dereference(sk->sk_wq); 2634 if (skwq_has_sleeper(wq)) 2635 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 2636 EPOLLRDNORM | EPOLLRDBAND); 2637 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 2638 rcu_read_unlock(); 2639 } 2640 2641 static void sock_def_write_space(struct sock *sk) 2642 { 2643 struct socket_wq *wq; 2644 2645 rcu_read_lock(); 2646 2647 /* Do not wake up a writer until he can make "significant" 2648 * progress. --DaveM 2649 */ 2650 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) { 2651 wq = rcu_dereference(sk->sk_wq); 2652 if (skwq_has_sleeper(wq)) 2653 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 2654 EPOLLWRNORM | EPOLLWRBAND); 2655 2656 /* Should agree with poll, otherwise some programs break */ 2657 if (sock_writeable(sk)) 2658 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 2659 } 2660 2661 rcu_read_unlock(); 2662 } 2663 2664 static void sock_def_destruct(struct sock *sk) 2665 { 2666 } 2667 2668 void sk_send_sigurg(struct sock *sk) 2669 { 2670 if (sk->sk_socket && sk->sk_socket->file) 2671 if (send_sigurg(&sk->sk_socket->file->f_owner)) 2672 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 2673 } 2674 EXPORT_SYMBOL(sk_send_sigurg); 2675 2676 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 2677 unsigned long expires) 2678 { 2679 if (!mod_timer(timer, expires)) 2680 sock_hold(sk); 2681 } 2682 EXPORT_SYMBOL(sk_reset_timer); 2683 2684 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 2685 { 2686 if (del_timer(timer)) 2687 __sock_put(sk); 2688 } 2689 EXPORT_SYMBOL(sk_stop_timer); 2690 2691 void sock_init_data(struct socket *sock, struct sock *sk) 2692 { 2693 sk_init_common(sk); 2694 sk->sk_send_head = NULL; 2695 2696 timer_setup(&sk->sk_timer, NULL, 0); 2697 2698 sk->sk_allocation = GFP_KERNEL; 2699 sk->sk_rcvbuf = sysctl_rmem_default; 2700 sk->sk_sndbuf = sysctl_wmem_default; 2701 sk->sk_state = TCP_CLOSE; 2702 sk_set_socket(sk, sock); 2703 2704 sock_set_flag(sk, SOCK_ZAPPED); 2705 2706 if (sock) { 2707 sk->sk_type = sock->type; 2708 sk->sk_wq = sock->wq; 2709 sock->sk = sk; 2710 sk->sk_uid = SOCK_INODE(sock)->i_uid; 2711 } else { 2712 sk->sk_wq = NULL; 2713 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0); 2714 } 2715 2716 rwlock_init(&sk->sk_callback_lock); 2717 if (sk->sk_kern_sock) 2718 lockdep_set_class_and_name( 2719 &sk->sk_callback_lock, 2720 af_kern_callback_keys + sk->sk_family, 2721 af_family_kern_clock_key_strings[sk->sk_family]); 2722 else 2723 lockdep_set_class_and_name( 2724 &sk->sk_callback_lock, 2725 af_callback_keys + sk->sk_family, 2726 af_family_clock_key_strings[sk->sk_family]); 2727 2728 sk->sk_state_change = sock_def_wakeup; 2729 sk->sk_data_ready = sock_def_readable; 2730 sk->sk_write_space = sock_def_write_space; 2731 sk->sk_error_report = sock_def_error_report; 2732 sk->sk_destruct = sock_def_destruct; 2733 2734 sk->sk_frag.page = NULL; 2735 sk->sk_frag.offset = 0; 2736 sk->sk_peek_off = -1; 2737 2738 sk->sk_peer_pid = NULL; 2739 sk->sk_peer_cred = NULL; 2740 sk->sk_write_pending = 0; 2741 sk->sk_rcvlowat = 1; 2742 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 2743 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 2744 2745 sk->sk_stamp = SK_DEFAULT_STAMP; 2746 atomic_set(&sk->sk_zckey, 0); 2747 2748 #ifdef CONFIG_NET_RX_BUSY_POLL 2749 sk->sk_napi_id = 0; 2750 sk->sk_ll_usec = sysctl_net_busy_read; 2751 #endif 2752 2753 sk->sk_max_pacing_rate = ~0UL; 2754 sk->sk_pacing_rate = ~0UL; 2755 sk->sk_pacing_shift = 10; 2756 sk->sk_incoming_cpu = -1; 2757 2758 sk_rx_queue_clear(sk); 2759 /* 2760 * Before updating sk_refcnt, we must commit prior changes to memory 2761 * (Documentation/RCU/rculist_nulls.txt for details) 2762 */ 2763 smp_wmb(); 2764 refcount_set(&sk->sk_refcnt, 1); 2765 atomic_set(&sk->sk_drops, 0); 2766 } 2767 EXPORT_SYMBOL(sock_init_data); 2768 2769 void lock_sock_nested(struct sock *sk, int subclass) 2770 { 2771 might_sleep(); 2772 spin_lock_bh(&sk->sk_lock.slock); 2773 if (sk->sk_lock.owned) 2774 __lock_sock(sk); 2775 sk->sk_lock.owned = 1; 2776 spin_unlock(&sk->sk_lock.slock); 2777 /* 2778 * The sk_lock has mutex_lock() semantics here: 2779 */ 2780 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 2781 local_bh_enable(); 2782 } 2783 EXPORT_SYMBOL(lock_sock_nested); 2784 2785 void release_sock(struct sock *sk) 2786 { 2787 spin_lock_bh(&sk->sk_lock.slock); 2788 if (sk->sk_backlog.tail) 2789 __release_sock(sk); 2790 2791 /* Warning : release_cb() might need to release sk ownership, 2792 * ie call sock_release_ownership(sk) before us. 2793 */ 2794 if (sk->sk_prot->release_cb) 2795 sk->sk_prot->release_cb(sk); 2796 2797 sock_release_ownership(sk); 2798 if (waitqueue_active(&sk->sk_lock.wq)) 2799 wake_up(&sk->sk_lock.wq); 2800 spin_unlock_bh(&sk->sk_lock.slock); 2801 } 2802 EXPORT_SYMBOL(release_sock); 2803 2804 /** 2805 * lock_sock_fast - fast version of lock_sock 2806 * @sk: socket 2807 * 2808 * This version should be used for very small section, where process wont block 2809 * return false if fast path is taken: 2810 * 2811 * sk_lock.slock locked, owned = 0, BH disabled 2812 * 2813 * return true if slow path is taken: 2814 * 2815 * sk_lock.slock unlocked, owned = 1, BH enabled 2816 */ 2817 bool lock_sock_fast(struct sock *sk) 2818 { 2819 might_sleep(); 2820 spin_lock_bh(&sk->sk_lock.slock); 2821 2822 if (!sk->sk_lock.owned) 2823 /* 2824 * Note : We must disable BH 2825 */ 2826 return false; 2827 2828 __lock_sock(sk); 2829 sk->sk_lock.owned = 1; 2830 spin_unlock(&sk->sk_lock.slock); 2831 /* 2832 * The sk_lock has mutex_lock() semantics here: 2833 */ 2834 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_); 2835 local_bh_enable(); 2836 return true; 2837 } 2838 EXPORT_SYMBOL(lock_sock_fast); 2839 2840 int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp) 2841 { 2842 struct timeval tv; 2843 2844 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2845 tv = ktime_to_timeval(sk->sk_stamp); 2846 if (tv.tv_sec == -1) 2847 return -ENOENT; 2848 if (tv.tv_sec == 0) { 2849 sk->sk_stamp = ktime_get_real(); 2850 tv = ktime_to_timeval(sk->sk_stamp); 2851 } 2852 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0; 2853 } 2854 EXPORT_SYMBOL(sock_get_timestamp); 2855 2856 int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp) 2857 { 2858 struct timespec ts; 2859 2860 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2861 ts = ktime_to_timespec(sk->sk_stamp); 2862 if (ts.tv_sec == -1) 2863 return -ENOENT; 2864 if (ts.tv_sec == 0) { 2865 sk->sk_stamp = ktime_get_real(); 2866 ts = ktime_to_timespec(sk->sk_stamp); 2867 } 2868 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0; 2869 } 2870 EXPORT_SYMBOL(sock_get_timestampns); 2871 2872 void sock_enable_timestamp(struct sock *sk, int flag) 2873 { 2874 if (!sock_flag(sk, flag)) { 2875 unsigned long previous_flags = sk->sk_flags; 2876 2877 sock_set_flag(sk, flag); 2878 /* 2879 * we just set one of the two flags which require net 2880 * time stamping, but time stamping might have been on 2881 * already because of the other one 2882 */ 2883 if (sock_needs_netstamp(sk) && 2884 !(previous_flags & SK_FLAGS_TIMESTAMP)) 2885 net_enable_timestamp(); 2886 } 2887 } 2888 2889 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 2890 int level, int type) 2891 { 2892 struct sock_exterr_skb *serr; 2893 struct sk_buff *skb; 2894 int copied, err; 2895 2896 err = -EAGAIN; 2897 skb = sock_dequeue_err_skb(sk); 2898 if (skb == NULL) 2899 goto out; 2900 2901 copied = skb->len; 2902 if (copied > len) { 2903 msg->msg_flags |= MSG_TRUNC; 2904 copied = len; 2905 } 2906 err = skb_copy_datagram_msg(skb, 0, msg, copied); 2907 if (err) 2908 goto out_free_skb; 2909 2910 sock_recv_timestamp(msg, sk, skb); 2911 2912 serr = SKB_EXT_ERR(skb); 2913 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 2914 2915 msg->msg_flags |= MSG_ERRQUEUE; 2916 err = copied; 2917 2918 out_free_skb: 2919 kfree_skb(skb); 2920 out: 2921 return err; 2922 } 2923 EXPORT_SYMBOL(sock_recv_errqueue); 2924 2925 /* 2926 * Get a socket option on an socket. 2927 * 2928 * FIX: POSIX 1003.1g is very ambiguous here. It states that 2929 * asynchronous errors should be reported by getsockopt. We assume 2930 * this means if you specify SO_ERROR (otherwise whats the point of it). 2931 */ 2932 int sock_common_getsockopt(struct socket *sock, int level, int optname, 2933 char __user *optval, int __user *optlen) 2934 { 2935 struct sock *sk = sock->sk; 2936 2937 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 2938 } 2939 EXPORT_SYMBOL(sock_common_getsockopt); 2940 2941 #ifdef CONFIG_COMPAT 2942 int compat_sock_common_getsockopt(struct socket *sock, int level, int optname, 2943 char __user *optval, int __user *optlen) 2944 { 2945 struct sock *sk = sock->sk; 2946 2947 if (sk->sk_prot->compat_getsockopt != NULL) 2948 return sk->sk_prot->compat_getsockopt(sk, level, optname, 2949 optval, optlen); 2950 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 2951 } 2952 EXPORT_SYMBOL(compat_sock_common_getsockopt); 2953 #endif 2954 2955 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 2956 int flags) 2957 { 2958 struct sock *sk = sock->sk; 2959 int addr_len = 0; 2960 int err; 2961 2962 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, 2963 flags & ~MSG_DONTWAIT, &addr_len); 2964 if (err >= 0) 2965 msg->msg_namelen = addr_len; 2966 return err; 2967 } 2968 EXPORT_SYMBOL(sock_common_recvmsg); 2969 2970 /* 2971 * Set socket options on an inet socket. 2972 */ 2973 int sock_common_setsockopt(struct socket *sock, int level, int optname, 2974 char __user *optval, unsigned int optlen) 2975 { 2976 struct sock *sk = sock->sk; 2977 2978 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 2979 } 2980 EXPORT_SYMBOL(sock_common_setsockopt); 2981 2982 #ifdef CONFIG_COMPAT 2983 int compat_sock_common_setsockopt(struct socket *sock, int level, int optname, 2984 char __user *optval, unsigned int optlen) 2985 { 2986 struct sock *sk = sock->sk; 2987 2988 if (sk->sk_prot->compat_setsockopt != NULL) 2989 return sk->sk_prot->compat_setsockopt(sk, level, optname, 2990 optval, optlen); 2991 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 2992 } 2993 EXPORT_SYMBOL(compat_sock_common_setsockopt); 2994 #endif 2995 2996 void sk_common_release(struct sock *sk) 2997 { 2998 if (sk->sk_prot->destroy) 2999 sk->sk_prot->destroy(sk); 3000 3001 /* 3002 * Observation: when sock_common_release is called, processes have 3003 * no access to socket. But net still has. 3004 * Step one, detach it from networking: 3005 * 3006 * A. Remove from hash tables. 3007 */ 3008 3009 sk->sk_prot->unhash(sk); 3010 3011 /* 3012 * In this point socket cannot receive new packets, but it is possible 3013 * that some packets are in flight because some CPU runs receiver and 3014 * did hash table lookup before we unhashed socket. They will achieve 3015 * receive queue and will be purged by socket destructor. 3016 * 3017 * Also we still have packets pending on receive queue and probably, 3018 * our own packets waiting in device queues. sock_destroy will drain 3019 * receive queue, but transmitted packets will delay socket destruction 3020 * until the last reference will be released. 3021 */ 3022 3023 sock_orphan(sk); 3024 3025 xfrm_sk_free_policy(sk); 3026 3027 sk_refcnt_debug_release(sk); 3028 3029 sock_put(sk); 3030 } 3031 EXPORT_SYMBOL(sk_common_release); 3032 3033 void sk_get_meminfo(const struct sock *sk, u32 *mem) 3034 { 3035 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 3036 3037 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 3038 mem[SK_MEMINFO_RCVBUF] = sk->sk_rcvbuf; 3039 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 3040 mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf; 3041 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc; 3042 mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued; 3043 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 3044 mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len; 3045 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops); 3046 } 3047 3048 #ifdef CONFIG_PROC_FS 3049 #define PROTO_INUSE_NR 64 /* should be enough for the first time */ 3050 struct prot_inuse { 3051 int val[PROTO_INUSE_NR]; 3052 }; 3053 3054 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 3055 3056 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) 3057 { 3058 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); 3059 } 3060 EXPORT_SYMBOL_GPL(sock_prot_inuse_add); 3061 3062 int sock_prot_inuse_get(struct net *net, struct proto *prot) 3063 { 3064 int cpu, idx = prot->inuse_idx; 3065 int res = 0; 3066 3067 for_each_possible_cpu(cpu) 3068 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 3069 3070 return res >= 0 ? res : 0; 3071 } 3072 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 3073 3074 static void sock_inuse_add(struct net *net, int val) 3075 { 3076 this_cpu_add(*net->core.sock_inuse, val); 3077 } 3078 3079 int sock_inuse_get(struct net *net) 3080 { 3081 int cpu, res = 0; 3082 3083 for_each_possible_cpu(cpu) 3084 res += *per_cpu_ptr(net->core.sock_inuse, cpu); 3085 3086 return res; 3087 } 3088 3089 EXPORT_SYMBOL_GPL(sock_inuse_get); 3090 3091 static int __net_init sock_inuse_init_net(struct net *net) 3092 { 3093 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 3094 if (net->core.prot_inuse == NULL) 3095 return -ENOMEM; 3096 3097 net->core.sock_inuse = alloc_percpu(int); 3098 if (net->core.sock_inuse == NULL) 3099 goto out; 3100 3101 return 0; 3102 3103 out: 3104 free_percpu(net->core.prot_inuse); 3105 return -ENOMEM; 3106 } 3107 3108 static void __net_exit sock_inuse_exit_net(struct net *net) 3109 { 3110 free_percpu(net->core.prot_inuse); 3111 free_percpu(net->core.sock_inuse); 3112 } 3113 3114 static struct pernet_operations net_inuse_ops = { 3115 .init = sock_inuse_init_net, 3116 .exit = sock_inuse_exit_net, 3117 }; 3118 3119 static __init int net_inuse_init(void) 3120 { 3121 if (register_pernet_subsys(&net_inuse_ops)) 3122 panic("Cannot initialize net inuse counters"); 3123 3124 return 0; 3125 } 3126 3127 core_initcall(net_inuse_init); 3128 3129 static void assign_proto_idx(struct proto *prot) 3130 { 3131 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 3132 3133 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 3134 pr_err("PROTO_INUSE_NR exhausted\n"); 3135 return; 3136 } 3137 3138 set_bit(prot->inuse_idx, proto_inuse_idx); 3139 } 3140 3141 static void release_proto_idx(struct proto *prot) 3142 { 3143 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 3144 clear_bit(prot->inuse_idx, proto_inuse_idx); 3145 } 3146 #else 3147 static inline void assign_proto_idx(struct proto *prot) 3148 { 3149 } 3150 3151 static inline void release_proto_idx(struct proto *prot) 3152 { 3153 } 3154 3155 static void sock_inuse_add(struct net *net, int val) 3156 { 3157 } 3158 #endif 3159 3160 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 3161 { 3162 if (!rsk_prot) 3163 return; 3164 kfree(rsk_prot->slab_name); 3165 rsk_prot->slab_name = NULL; 3166 kmem_cache_destroy(rsk_prot->slab); 3167 rsk_prot->slab = NULL; 3168 } 3169 3170 static int req_prot_init(const struct proto *prot) 3171 { 3172 struct request_sock_ops *rsk_prot = prot->rsk_prot; 3173 3174 if (!rsk_prot) 3175 return 0; 3176 3177 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 3178 prot->name); 3179 if (!rsk_prot->slab_name) 3180 return -ENOMEM; 3181 3182 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 3183 rsk_prot->obj_size, 0, 3184 SLAB_ACCOUNT | prot->slab_flags, 3185 NULL); 3186 3187 if (!rsk_prot->slab) { 3188 pr_crit("%s: Can't create request sock SLAB cache!\n", 3189 prot->name); 3190 return -ENOMEM; 3191 } 3192 return 0; 3193 } 3194 3195 int proto_register(struct proto *prot, int alloc_slab) 3196 { 3197 if (alloc_slab) { 3198 prot->slab = kmem_cache_create_usercopy(prot->name, 3199 prot->obj_size, 0, 3200 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 3201 prot->slab_flags, 3202 prot->useroffset, prot->usersize, 3203 NULL); 3204 3205 if (prot->slab == NULL) { 3206 pr_crit("%s: Can't create sock SLAB cache!\n", 3207 prot->name); 3208 goto out; 3209 } 3210 3211 if (req_prot_init(prot)) 3212 goto out_free_request_sock_slab; 3213 3214 if (prot->twsk_prot != NULL) { 3215 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name); 3216 3217 if (prot->twsk_prot->twsk_slab_name == NULL) 3218 goto out_free_request_sock_slab; 3219 3220 prot->twsk_prot->twsk_slab = 3221 kmem_cache_create(prot->twsk_prot->twsk_slab_name, 3222 prot->twsk_prot->twsk_obj_size, 3223 0, 3224 SLAB_ACCOUNT | 3225 prot->slab_flags, 3226 NULL); 3227 if (prot->twsk_prot->twsk_slab == NULL) 3228 goto out_free_timewait_sock_slab_name; 3229 } 3230 } 3231 3232 mutex_lock(&proto_list_mutex); 3233 list_add(&prot->node, &proto_list); 3234 assign_proto_idx(prot); 3235 mutex_unlock(&proto_list_mutex); 3236 return 0; 3237 3238 out_free_timewait_sock_slab_name: 3239 kfree(prot->twsk_prot->twsk_slab_name); 3240 out_free_request_sock_slab: 3241 req_prot_cleanup(prot->rsk_prot); 3242 3243 kmem_cache_destroy(prot->slab); 3244 prot->slab = NULL; 3245 out: 3246 return -ENOBUFS; 3247 } 3248 EXPORT_SYMBOL(proto_register); 3249 3250 void proto_unregister(struct proto *prot) 3251 { 3252 mutex_lock(&proto_list_mutex); 3253 release_proto_idx(prot); 3254 list_del(&prot->node); 3255 mutex_unlock(&proto_list_mutex); 3256 3257 kmem_cache_destroy(prot->slab); 3258 prot->slab = NULL; 3259 3260 req_prot_cleanup(prot->rsk_prot); 3261 3262 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) { 3263 kmem_cache_destroy(prot->twsk_prot->twsk_slab); 3264 kfree(prot->twsk_prot->twsk_slab_name); 3265 prot->twsk_prot->twsk_slab = NULL; 3266 } 3267 } 3268 EXPORT_SYMBOL(proto_unregister); 3269 3270 int sock_load_diag_module(int family, int protocol) 3271 { 3272 if (!protocol) { 3273 if (!sock_is_registered(family)) 3274 return -ENOENT; 3275 3276 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 3277 NETLINK_SOCK_DIAG, family); 3278 } 3279 3280 #ifdef CONFIG_INET 3281 if (family == AF_INET && 3282 !rcu_access_pointer(inet_protos[protocol])) 3283 return -ENOENT; 3284 #endif 3285 3286 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 3287 NETLINK_SOCK_DIAG, family, protocol); 3288 } 3289 EXPORT_SYMBOL(sock_load_diag_module); 3290 3291 #ifdef CONFIG_PROC_FS 3292 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 3293 __acquires(proto_list_mutex) 3294 { 3295 mutex_lock(&proto_list_mutex); 3296 return seq_list_start_head(&proto_list, *pos); 3297 } 3298 3299 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3300 { 3301 return seq_list_next(v, &proto_list, pos); 3302 } 3303 3304 static void proto_seq_stop(struct seq_file *seq, void *v) 3305 __releases(proto_list_mutex) 3306 { 3307 mutex_unlock(&proto_list_mutex); 3308 } 3309 3310 static char proto_method_implemented(const void *method) 3311 { 3312 return method == NULL ? 'n' : 'y'; 3313 } 3314 static long sock_prot_memory_allocated(struct proto *proto) 3315 { 3316 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 3317 } 3318 3319 static char *sock_prot_memory_pressure(struct proto *proto) 3320 { 3321 return proto->memory_pressure != NULL ? 3322 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 3323 } 3324 3325 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 3326 { 3327 3328 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 3329 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 3330 proto->name, 3331 proto->obj_size, 3332 sock_prot_inuse_get(seq_file_net(seq), proto), 3333 sock_prot_memory_allocated(proto), 3334 sock_prot_memory_pressure(proto), 3335 proto->max_header, 3336 proto->slab == NULL ? "no" : "yes", 3337 module_name(proto->owner), 3338 proto_method_implemented(proto->close), 3339 proto_method_implemented(proto->connect), 3340 proto_method_implemented(proto->disconnect), 3341 proto_method_implemented(proto->accept), 3342 proto_method_implemented(proto->ioctl), 3343 proto_method_implemented(proto->init), 3344 proto_method_implemented(proto->destroy), 3345 proto_method_implemented(proto->shutdown), 3346 proto_method_implemented(proto->setsockopt), 3347 proto_method_implemented(proto->getsockopt), 3348 proto_method_implemented(proto->sendmsg), 3349 proto_method_implemented(proto->recvmsg), 3350 proto_method_implemented(proto->sendpage), 3351 proto_method_implemented(proto->bind), 3352 proto_method_implemented(proto->backlog_rcv), 3353 proto_method_implemented(proto->hash), 3354 proto_method_implemented(proto->unhash), 3355 proto_method_implemented(proto->get_port), 3356 proto_method_implemented(proto->enter_memory_pressure)); 3357 } 3358 3359 static int proto_seq_show(struct seq_file *seq, void *v) 3360 { 3361 if (v == &proto_list) 3362 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 3363 "protocol", 3364 "size", 3365 "sockets", 3366 "memory", 3367 "press", 3368 "maxhdr", 3369 "slab", 3370 "module", 3371 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n"); 3372 else 3373 proto_seq_printf(seq, list_entry(v, struct proto, node)); 3374 return 0; 3375 } 3376 3377 static const struct seq_operations proto_seq_ops = { 3378 .start = proto_seq_start, 3379 .next = proto_seq_next, 3380 .stop = proto_seq_stop, 3381 .show = proto_seq_show, 3382 }; 3383 3384 static __net_init int proto_init_net(struct net *net) 3385 { 3386 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 3387 sizeof(struct seq_net_private))) 3388 return -ENOMEM; 3389 3390 return 0; 3391 } 3392 3393 static __net_exit void proto_exit_net(struct net *net) 3394 { 3395 remove_proc_entry("protocols", net->proc_net); 3396 } 3397 3398 3399 static __net_initdata struct pernet_operations proto_net_ops = { 3400 .init = proto_init_net, 3401 .exit = proto_exit_net, 3402 }; 3403 3404 static int __init proto_init(void) 3405 { 3406 return register_pernet_subsys(&proto_net_ops); 3407 } 3408 3409 subsys_initcall(proto_init); 3410 3411 #endif /* PROC_FS */ 3412 3413 #ifdef CONFIG_NET_RX_BUSY_POLL 3414 bool sk_busy_loop_end(void *p, unsigned long start_time) 3415 { 3416 struct sock *sk = p; 3417 3418 return !skb_queue_empty(&sk->sk_receive_queue) || 3419 sk_busy_loop_timeout(sk, start_time); 3420 } 3421 EXPORT_SYMBOL(sk_busy_loop_end); 3422 #endif /* CONFIG_NET_RX_BUSY_POLL */ 3423