1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Generic socket support routines. Memory allocators, socket lock/release 7 * handler for protocols to use and generic option handler. 8 * 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 * 85 * 86 * This program is free software; you can redistribute it and/or 87 * modify it under the terms of the GNU General Public License 88 * as published by the Free Software Foundation; either version 89 * 2 of the License, or (at your option) any later version. 90 */ 91 92 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 93 94 #include <asm/unaligned.h> 95 #include <linux/capability.h> 96 #include <linux/errno.h> 97 #include <linux/errqueue.h> 98 #include <linux/types.h> 99 #include <linux/socket.h> 100 #include <linux/in.h> 101 #include <linux/kernel.h> 102 #include <linux/module.h> 103 #include <linux/proc_fs.h> 104 #include <linux/seq_file.h> 105 #include <linux/sched.h> 106 #include <linux/sched/mm.h> 107 #include <linux/timer.h> 108 #include <linux/string.h> 109 #include <linux/sockios.h> 110 #include <linux/net.h> 111 #include <linux/mm.h> 112 #include <linux/slab.h> 113 #include <linux/interrupt.h> 114 #include <linux/poll.h> 115 #include <linux/tcp.h> 116 #include <linux/init.h> 117 #include <linux/highmem.h> 118 #include <linux/user_namespace.h> 119 #include <linux/static_key.h> 120 #include <linux/memcontrol.h> 121 #include <linux/prefetch.h> 122 123 #include <linux/uaccess.h> 124 125 #include <linux/netdevice.h> 126 #include <net/protocol.h> 127 #include <linux/skbuff.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <linux/net_tstamp.h> 132 #include <net/xfrm.h> 133 #include <linux/ipsec.h> 134 #include <net/cls_cgroup.h> 135 #include <net/netprio_cgroup.h> 136 #include <linux/sock_diag.h> 137 138 #include <linux/filter.h> 139 #include <net/sock_reuseport.h> 140 141 #include <trace/events/sock.h> 142 143 #include <net/tcp.h> 144 #include <net/busy_poll.h> 145 146 static DEFINE_MUTEX(proto_list_mutex); 147 static LIST_HEAD(proto_list); 148 149 static void sock_inuse_add(struct net *net, int val); 150 151 /** 152 * sk_ns_capable - General socket capability test 153 * @sk: Socket to use a capability on or through 154 * @user_ns: The user namespace of the capability to use 155 * @cap: The capability to use 156 * 157 * Test to see if the opener of the socket had when the socket was 158 * created and the current process has the capability @cap in the user 159 * namespace @user_ns. 160 */ 161 bool sk_ns_capable(const struct sock *sk, 162 struct user_namespace *user_ns, int cap) 163 { 164 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 165 ns_capable(user_ns, cap); 166 } 167 EXPORT_SYMBOL(sk_ns_capable); 168 169 /** 170 * sk_capable - Socket global capability test 171 * @sk: Socket to use a capability on or through 172 * @cap: The global capability to use 173 * 174 * Test to see if the opener of the socket had when the socket was 175 * created and the current process has the capability @cap in all user 176 * namespaces. 177 */ 178 bool sk_capable(const struct sock *sk, int cap) 179 { 180 return sk_ns_capable(sk, &init_user_ns, cap); 181 } 182 EXPORT_SYMBOL(sk_capable); 183 184 /** 185 * sk_net_capable - Network namespace socket capability test 186 * @sk: Socket to use a capability on or through 187 * @cap: The capability to use 188 * 189 * Test to see if the opener of the socket had when the socket was created 190 * and the current process has the capability @cap over the network namespace 191 * the socket is a member of. 192 */ 193 bool sk_net_capable(const struct sock *sk, int cap) 194 { 195 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 196 } 197 EXPORT_SYMBOL(sk_net_capable); 198 199 /* 200 * Each address family might have different locking rules, so we have 201 * one slock key per address family and separate keys for internal and 202 * userspace sockets. 203 */ 204 static struct lock_class_key af_family_keys[AF_MAX]; 205 static struct lock_class_key af_family_kern_keys[AF_MAX]; 206 static struct lock_class_key af_family_slock_keys[AF_MAX]; 207 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 208 209 /* 210 * Make lock validator output more readable. (we pre-construct these 211 * strings build-time, so that runtime initialization of socket 212 * locks is fast): 213 */ 214 215 #define _sock_locks(x) \ 216 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 217 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 218 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 219 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 220 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 221 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 222 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 223 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 224 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 225 x "27" , x "28" , x "AF_CAN" , \ 226 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 227 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 228 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 229 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 230 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 231 x "AF_MAX" 232 233 static const char *const af_family_key_strings[AF_MAX+1] = { 234 _sock_locks("sk_lock-") 235 }; 236 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 237 _sock_locks("slock-") 238 }; 239 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 240 _sock_locks("clock-") 241 }; 242 243 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 244 _sock_locks("k-sk_lock-") 245 }; 246 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 247 _sock_locks("k-slock-") 248 }; 249 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 250 _sock_locks("k-clock-") 251 }; 252 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 253 _sock_locks("rlock-") 254 }; 255 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 256 _sock_locks("wlock-") 257 }; 258 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 259 _sock_locks("elock-") 260 }; 261 262 /* 263 * sk_callback_lock and sk queues locking rules are per-address-family, 264 * so split the lock classes by using a per-AF key: 265 */ 266 static struct lock_class_key af_callback_keys[AF_MAX]; 267 static struct lock_class_key af_rlock_keys[AF_MAX]; 268 static struct lock_class_key af_wlock_keys[AF_MAX]; 269 static struct lock_class_key af_elock_keys[AF_MAX]; 270 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 271 272 /* Run time adjustable parameters. */ 273 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 274 EXPORT_SYMBOL(sysctl_wmem_max); 275 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 276 EXPORT_SYMBOL(sysctl_rmem_max); 277 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 278 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 279 280 /* Maximal space eaten by iovec or ancillary data plus some space */ 281 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 282 EXPORT_SYMBOL(sysctl_optmem_max); 283 284 int sysctl_tstamp_allow_data __read_mostly = 1; 285 286 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 287 EXPORT_SYMBOL_GPL(memalloc_socks_key); 288 289 /** 290 * sk_set_memalloc - sets %SOCK_MEMALLOC 291 * @sk: socket to set it on 292 * 293 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 294 * It's the responsibility of the admin to adjust min_free_kbytes 295 * to meet the requirements 296 */ 297 void sk_set_memalloc(struct sock *sk) 298 { 299 sock_set_flag(sk, SOCK_MEMALLOC); 300 sk->sk_allocation |= __GFP_MEMALLOC; 301 static_branch_inc(&memalloc_socks_key); 302 } 303 EXPORT_SYMBOL_GPL(sk_set_memalloc); 304 305 void sk_clear_memalloc(struct sock *sk) 306 { 307 sock_reset_flag(sk, SOCK_MEMALLOC); 308 sk->sk_allocation &= ~__GFP_MEMALLOC; 309 static_branch_dec(&memalloc_socks_key); 310 311 /* 312 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 313 * progress of swapping. SOCK_MEMALLOC may be cleared while 314 * it has rmem allocations due to the last swapfile being deactivated 315 * but there is a risk that the socket is unusable due to exceeding 316 * the rmem limits. Reclaim the reserves and obey rmem limits again. 317 */ 318 sk_mem_reclaim(sk); 319 } 320 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 321 322 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 323 { 324 int ret; 325 unsigned int noreclaim_flag; 326 327 /* these should have been dropped before queueing */ 328 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 329 330 noreclaim_flag = memalloc_noreclaim_save(); 331 ret = sk->sk_backlog_rcv(sk, skb); 332 memalloc_noreclaim_restore(noreclaim_flag); 333 334 return ret; 335 } 336 EXPORT_SYMBOL(__sk_backlog_rcv); 337 338 static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen) 339 { 340 struct timeval tv; 341 342 if (optlen < sizeof(tv)) 343 return -EINVAL; 344 if (copy_from_user(&tv, optval, sizeof(tv))) 345 return -EFAULT; 346 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 347 return -EDOM; 348 349 if (tv.tv_sec < 0) { 350 static int warned __read_mostly; 351 352 *timeo_p = 0; 353 if (warned < 10 && net_ratelimit()) { 354 warned++; 355 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 356 __func__, current->comm, task_pid_nr(current)); 357 } 358 return 0; 359 } 360 *timeo_p = MAX_SCHEDULE_TIMEOUT; 361 if (tv.tv_sec == 0 && tv.tv_usec == 0) 362 return 0; 363 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1)) 364 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ); 365 return 0; 366 } 367 368 static void sock_warn_obsolete_bsdism(const char *name) 369 { 370 static int warned; 371 static char warncomm[TASK_COMM_LEN]; 372 if (strcmp(warncomm, current->comm) && warned < 5) { 373 strcpy(warncomm, current->comm); 374 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n", 375 warncomm, name); 376 warned++; 377 } 378 } 379 380 static bool sock_needs_netstamp(const struct sock *sk) 381 { 382 switch (sk->sk_family) { 383 case AF_UNSPEC: 384 case AF_UNIX: 385 return false; 386 default: 387 return true; 388 } 389 } 390 391 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 392 { 393 if (sk->sk_flags & flags) { 394 sk->sk_flags &= ~flags; 395 if (sock_needs_netstamp(sk) && 396 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 397 net_disable_timestamp(); 398 } 399 } 400 401 402 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 403 { 404 unsigned long flags; 405 struct sk_buff_head *list = &sk->sk_receive_queue; 406 407 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { 408 atomic_inc(&sk->sk_drops); 409 trace_sock_rcvqueue_full(sk, skb); 410 return -ENOMEM; 411 } 412 413 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 414 atomic_inc(&sk->sk_drops); 415 return -ENOBUFS; 416 } 417 418 skb->dev = NULL; 419 skb_set_owner_r(skb, sk); 420 421 /* we escape from rcu protected region, make sure we dont leak 422 * a norefcounted dst 423 */ 424 skb_dst_force(skb); 425 426 spin_lock_irqsave(&list->lock, flags); 427 sock_skb_set_dropcount(sk, skb); 428 __skb_queue_tail(list, skb); 429 spin_unlock_irqrestore(&list->lock, flags); 430 431 if (!sock_flag(sk, SOCK_DEAD)) 432 sk->sk_data_ready(sk); 433 return 0; 434 } 435 EXPORT_SYMBOL(__sock_queue_rcv_skb); 436 437 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 438 { 439 int err; 440 441 err = sk_filter(sk, skb); 442 if (err) 443 return err; 444 445 return __sock_queue_rcv_skb(sk, skb); 446 } 447 EXPORT_SYMBOL(sock_queue_rcv_skb); 448 449 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 450 const int nested, unsigned int trim_cap, bool refcounted) 451 { 452 int rc = NET_RX_SUCCESS; 453 454 if (sk_filter_trim_cap(sk, skb, trim_cap)) 455 goto discard_and_relse; 456 457 skb->dev = NULL; 458 459 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) { 460 atomic_inc(&sk->sk_drops); 461 goto discard_and_relse; 462 } 463 if (nested) 464 bh_lock_sock_nested(sk); 465 else 466 bh_lock_sock(sk); 467 if (!sock_owned_by_user(sk)) { 468 /* 469 * trylock + unlock semantics: 470 */ 471 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 472 473 rc = sk_backlog_rcv(sk, skb); 474 475 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_); 476 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) { 477 bh_unlock_sock(sk); 478 atomic_inc(&sk->sk_drops); 479 goto discard_and_relse; 480 } 481 482 bh_unlock_sock(sk); 483 out: 484 if (refcounted) 485 sock_put(sk); 486 return rc; 487 discard_and_relse: 488 kfree_skb(skb); 489 goto out; 490 } 491 EXPORT_SYMBOL(__sk_receive_skb); 492 493 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 494 { 495 struct dst_entry *dst = __sk_dst_get(sk); 496 497 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 498 sk_tx_queue_clear(sk); 499 sk->sk_dst_pending_confirm = 0; 500 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 501 dst_release(dst); 502 return NULL; 503 } 504 505 return dst; 506 } 507 EXPORT_SYMBOL(__sk_dst_check); 508 509 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 510 { 511 struct dst_entry *dst = sk_dst_get(sk); 512 513 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 514 sk_dst_reset(sk); 515 dst_release(dst); 516 return NULL; 517 } 518 519 return dst; 520 } 521 EXPORT_SYMBOL(sk_dst_check); 522 523 static int sock_setbindtodevice(struct sock *sk, char __user *optval, 524 int optlen) 525 { 526 int ret = -ENOPROTOOPT; 527 #ifdef CONFIG_NETDEVICES 528 struct net *net = sock_net(sk); 529 char devname[IFNAMSIZ]; 530 int index; 531 532 /* Sorry... */ 533 ret = -EPERM; 534 if (!ns_capable(net->user_ns, CAP_NET_RAW)) 535 goto out; 536 537 ret = -EINVAL; 538 if (optlen < 0) 539 goto out; 540 541 /* Bind this socket to a particular device like "eth0", 542 * as specified in the passed interface name. If the 543 * name is "" or the option length is zero the socket 544 * is not bound. 545 */ 546 if (optlen > IFNAMSIZ - 1) 547 optlen = IFNAMSIZ - 1; 548 memset(devname, 0, sizeof(devname)); 549 550 ret = -EFAULT; 551 if (copy_from_user(devname, optval, optlen)) 552 goto out; 553 554 index = 0; 555 if (devname[0] != '\0') { 556 struct net_device *dev; 557 558 rcu_read_lock(); 559 dev = dev_get_by_name_rcu(net, devname); 560 if (dev) 561 index = dev->ifindex; 562 rcu_read_unlock(); 563 ret = -ENODEV; 564 if (!dev) 565 goto out; 566 } 567 568 lock_sock(sk); 569 sk->sk_bound_dev_if = index; 570 sk_dst_reset(sk); 571 release_sock(sk); 572 573 ret = 0; 574 575 out: 576 #endif 577 578 return ret; 579 } 580 581 static int sock_getbindtodevice(struct sock *sk, char __user *optval, 582 int __user *optlen, int len) 583 { 584 int ret = -ENOPROTOOPT; 585 #ifdef CONFIG_NETDEVICES 586 struct net *net = sock_net(sk); 587 char devname[IFNAMSIZ]; 588 589 if (sk->sk_bound_dev_if == 0) { 590 len = 0; 591 goto zero; 592 } 593 594 ret = -EINVAL; 595 if (len < IFNAMSIZ) 596 goto out; 597 598 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if); 599 if (ret) 600 goto out; 601 602 len = strlen(devname) + 1; 603 604 ret = -EFAULT; 605 if (copy_to_user(optval, devname, len)) 606 goto out; 607 608 zero: 609 ret = -EFAULT; 610 if (put_user(len, optlen)) 611 goto out; 612 613 ret = 0; 614 615 out: 616 #endif 617 618 return ret; 619 } 620 621 static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool) 622 { 623 if (valbool) 624 sock_set_flag(sk, bit); 625 else 626 sock_reset_flag(sk, bit); 627 } 628 629 bool sk_mc_loop(struct sock *sk) 630 { 631 if (dev_recursion_level()) 632 return false; 633 if (!sk) 634 return true; 635 switch (sk->sk_family) { 636 case AF_INET: 637 return inet_sk(sk)->mc_loop; 638 #if IS_ENABLED(CONFIG_IPV6) 639 case AF_INET6: 640 return inet6_sk(sk)->mc_loop; 641 #endif 642 } 643 WARN_ON(1); 644 return true; 645 } 646 EXPORT_SYMBOL(sk_mc_loop); 647 648 /* 649 * This is meant for all protocols to use and covers goings on 650 * at the socket level. Everything here is generic. 651 */ 652 653 int sock_setsockopt(struct socket *sock, int level, int optname, 654 char __user *optval, unsigned int optlen) 655 { 656 struct sock_txtime sk_txtime; 657 struct sock *sk = sock->sk; 658 int val; 659 int valbool; 660 struct linger ling; 661 int ret = 0; 662 663 /* 664 * Options without arguments 665 */ 666 667 if (optname == SO_BINDTODEVICE) 668 return sock_setbindtodevice(sk, optval, optlen); 669 670 if (optlen < sizeof(int)) 671 return -EINVAL; 672 673 if (get_user(val, (int __user *)optval)) 674 return -EFAULT; 675 676 valbool = val ? 1 : 0; 677 678 lock_sock(sk); 679 680 switch (optname) { 681 case SO_DEBUG: 682 if (val && !capable(CAP_NET_ADMIN)) 683 ret = -EACCES; 684 else 685 sock_valbool_flag(sk, SOCK_DBG, valbool); 686 break; 687 case SO_REUSEADDR: 688 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 689 break; 690 case SO_REUSEPORT: 691 sk->sk_reuseport = valbool; 692 break; 693 case SO_TYPE: 694 case SO_PROTOCOL: 695 case SO_DOMAIN: 696 case SO_ERROR: 697 ret = -ENOPROTOOPT; 698 break; 699 case SO_DONTROUTE: 700 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 701 break; 702 case SO_BROADCAST: 703 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 704 break; 705 case SO_SNDBUF: 706 /* Don't error on this BSD doesn't and if you think 707 * about it this is right. Otherwise apps have to 708 * play 'guess the biggest size' games. RCVBUF/SNDBUF 709 * are treated in BSD as hints 710 */ 711 val = min_t(u32, val, sysctl_wmem_max); 712 set_sndbuf: 713 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 714 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); 715 /* Wake up sending tasks if we upped the value. */ 716 sk->sk_write_space(sk); 717 break; 718 719 case SO_SNDBUFFORCE: 720 if (!capable(CAP_NET_ADMIN)) { 721 ret = -EPERM; 722 break; 723 } 724 goto set_sndbuf; 725 726 case SO_RCVBUF: 727 /* Don't error on this BSD doesn't and if you think 728 * about it this is right. Otherwise apps have to 729 * play 'guess the biggest size' games. RCVBUF/SNDBUF 730 * are treated in BSD as hints 731 */ 732 val = min_t(u32, val, sysctl_rmem_max); 733 set_rcvbuf: 734 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 735 /* 736 * We double it on the way in to account for 737 * "struct sk_buff" etc. overhead. Applications 738 * assume that the SO_RCVBUF setting they make will 739 * allow that much actual data to be received on that 740 * socket. 741 * 742 * Applications are unaware that "struct sk_buff" and 743 * other overheads allocate from the receive buffer 744 * during socket buffer allocation. 745 * 746 * And after considering the possible alternatives, 747 * returning the value we actually used in getsockopt 748 * is the most desirable behavior. 749 */ 750 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF); 751 break; 752 753 case SO_RCVBUFFORCE: 754 if (!capable(CAP_NET_ADMIN)) { 755 ret = -EPERM; 756 break; 757 } 758 goto set_rcvbuf; 759 760 case SO_KEEPALIVE: 761 if (sk->sk_prot->keepalive) 762 sk->sk_prot->keepalive(sk, valbool); 763 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 764 break; 765 766 case SO_OOBINLINE: 767 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 768 break; 769 770 case SO_NO_CHECK: 771 sk->sk_no_check_tx = valbool; 772 break; 773 774 case SO_PRIORITY: 775 if ((val >= 0 && val <= 6) || 776 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 777 sk->sk_priority = val; 778 else 779 ret = -EPERM; 780 break; 781 782 case SO_LINGER: 783 if (optlen < sizeof(ling)) { 784 ret = -EINVAL; /* 1003.1g */ 785 break; 786 } 787 if (copy_from_user(&ling, optval, sizeof(ling))) { 788 ret = -EFAULT; 789 break; 790 } 791 if (!ling.l_onoff) 792 sock_reset_flag(sk, SOCK_LINGER); 793 else { 794 #if (BITS_PER_LONG == 32) 795 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ) 796 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT; 797 else 798 #endif 799 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ; 800 sock_set_flag(sk, SOCK_LINGER); 801 } 802 break; 803 804 case SO_BSDCOMPAT: 805 sock_warn_obsolete_bsdism("setsockopt"); 806 break; 807 808 case SO_PASSCRED: 809 if (valbool) 810 set_bit(SOCK_PASSCRED, &sock->flags); 811 else 812 clear_bit(SOCK_PASSCRED, &sock->flags); 813 break; 814 815 case SO_TIMESTAMP: 816 case SO_TIMESTAMPNS: 817 if (valbool) { 818 if (optname == SO_TIMESTAMP) 819 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 820 else 821 sock_set_flag(sk, SOCK_RCVTSTAMPNS); 822 sock_set_flag(sk, SOCK_RCVTSTAMP); 823 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 824 } else { 825 sock_reset_flag(sk, SOCK_RCVTSTAMP); 826 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 827 } 828 break; 829 830 case SO_TIMESTAMPING: 831 if (val & ~SOF_TIMESTAMPING_MASK) { 832 ret = -EINVAL; 833 break; 834 } 835 836 if (val & SOF_TIMESTAMPING_OPT_ID && 837 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 838 if (sk->sk_protocol == IPPROTO_TCP && 839 sk->sk_type == SOCK_STREAM) { 840 if ((1 << sk->sk_state) & 841 (TCPF_CLOSE | TCPF_LISTEN)) { 842 ret = -EINVAL; 843 break; 844 } 845 sk->sk_tskey = tcp_sk(sk)->snd_una; 846 } else { 847 sk->sk_tskey = 0; 848 } 849 } 850 851 if (val & SOF_TIMESTAMPING_OPT_STATS && 852 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) { 853 ret = -EINVAL; 854 break; 855 } 856 857 sk->sk_tsflags = val; 858 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 859 sock_enable_timestamp(sk, 860 SOCK_TIMESTAMPING_RX_SOFTWARE); 861 else 862 sock_disable_timestamp(sk, 863 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 864 break; 865 866 case SO_RCVLOWAT: 867 if (val < 0) 868 val = INT_MAX; 869 if (sock->ops->set_rcvlowat) 870 ret = sock->ops->set_rcvlowat(sk, val); 871 else 872 sk->sk_rcvlowat = val ? : 1; 873 break; 874 875 case SO_RCVTIMEO: 876 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen); 877 break; 878 879 case SO_SNDTIMEO: 880 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen); 881 break; 882 883 case SO_ATTACH_FILTER: 884 ret = -EINVAL; 885 if (optlen == sizeof(struct sock_fprog)) { 886 struct sock_fprog fprog; 887 888 ret = -EFAULT; 889 if (copy_from_user(&fprog, optval, sizeof(fprog))) 890 break; 891 892 ret = sk_attach_filter(&fprog, sk); 893 } 894 break; 895 896 case SO_ATTACH_BPF: 897 ret = -EINVAL; 898 if (optlen == sizeof(u32)) { 899 u32 ufd; 900 901 ret = -EFAULT; 902 if (copy_from_user(&ufd, optval, sizeof(ufd))) 903 break; 904 905 ret = sk_attach_bpf(ufd, sk); 906 } 907 break; 908 909 case SO_ATTACH_REUSEPORT_CBPF: 910 ret = -EINVAL; 911 if (optlen == sizeof(struct sock_fprog)) { 912 struct sock_fprog fprog; 913 914 ret = -EFAULT; 915 if (copy_from_user(&fprog, optval, sizeof(fprog))) 916 break; 917 918 ret = sk_reuseport_attach_filter(&fprog, sk); 919 } 920 break; 921 922 case SO_ATTACH_REUSEPORT_EBPF: 923 ret = -EINVAL; 924 if (optlen == sizeof(u32)) { 925 u32 ufd; 926 927 ret = -EFAULT; 928 if (copy_from_user(&ufd, optval, sizeof(ufd))) 929 break; 930 931 ret = sk_reuseport_attach_bpf(ufd, sk); 932 } 933 break; 934 935 case SO_DETACH_FILTER: 936 ret = sk_detach_filter(sk); 937 break; 938 939 case SO_LOCK_FILTER: 940 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 941 ret = -EPERM; 942 else 943 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 944 break; 945 946 case SO_PASSSEC: 947 if (valbool) 948 set_bit(SOCK_PASSSEC, &sock->flags); 949 else 950 clear_bit(SOCK_PASSSEC, &sock->flags); 951 break; 952 case SO_MARK: 953 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 954 ret = -EPERM; 955 else 956 sk->sk_mark = val; 957 break; 958 959 case SO_RXQ_OVFL: 960 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 961 break; 962 963 case SO_WIFI_STATUS: 964 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 965 break; 966 967 case SO_PEEK_OFF: 968 if (sock->ops->set_peek_off) 969 ret = sock->ops->set_peek_off(sk, val); 970 else 971 ret = -EOPNOTSUPP; 972 break; 973 974 case SO_NOFCS: 975 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 976 break; 977 978 case SO_SELECT_ERR_QUEUE: 979 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 980 break; 981 982 #ifdef CONFIG_NET_RX_BUSY_POLL 983 case SO_BUSY_POLL: 984 /* allow unprivileged users to decrease the value */ 985 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN)) 986 ret = -EPERM; 987 else { 988 if (val < 0) 989 ret = -EINVAL; 990 else 991 sk->sk_ll_usec = val; 992 } 993 break; 994 #endif 995 996 case SO_MAX_PACING_RATE: 997 if (val != ~0U) 998 cmpxchg(&sk->sk_pacing_status, 999 SK_PACING_NONE, 1000 SK_PACING_NEEDED); 1001 sk->sk_max_pacing_rate = (val == ~0U) ? ~0UL : val; 1002 sk->sk_pacing_rate = min(sk->sk_pacing_rate, 1003 sk->sk_max_pacing_rate); 1004 break; 1005 1006 case SO_INCOMING_CPU: 1007 sk->sk_incoming_cpu = val; 1008 break; 1009 1010 case SO_CNX_ADVICE: 1011 if (val == 1) 1012 dst_negative_advice(sk); 1013 break; 1014 1015 case SO_ZEROCOPY: 1016 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1017 if (sk->sk_protocol != IPPROTO_TCP) 1018 ret = -ENOTSUPP; 1019 } else if (sk->sk_family != PF_RDS) { 1020 ret = -ENOTSUPP; 1021 } 1022 if (!ret) { 1023 if (val < 0 || val > 1) 1024 ret = -EINVAL; 1025 else 1026 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1027 } 1028 break; 1029 1030 case SO_TXTIME: 1031 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1032 ret = -EPERM; 1033 } else if (optlen != sizeof(struct sock_txtime)) { 1034 ret = -EINVAL; 1035 } else if (copy_from_user(&sk_txtime, optval, 1036 sizeof(struct sock_txtime))) { 1037 ret = -EFAULT; 1038 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1039 ret = -EINVAL; 1040 } else { 1041 sock_valbool_flag(sk, SOCK_TXTIME, true); 1042 sk->sk_clockid = sk_txtime.clockid; 1043 sk->sk_txtime_deadline_mode = 1044 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1045 sk->sk_txtime_report_errors = 1046 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1047 } 1048 break; 1049 1050 default: 1051 ret = -ENOPROTOOPT; 1052 break; 1053 } 1054 release_sock(sk); 1055 return ret; 1056 } 1057 EXPORT_SYMBOL(sock_setsockopt); 1058 1059 1060 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1061 struct ucred *ucred) 1062 { 1063 ucred->pid = pid_vnr(pid); 1064 ucred->uid = ucred->gid = -1; 1065 if (cred) { 1066 struct user_namespace *current_ns = current_user_ns(); 1067 1068 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1069 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1070 } 1071 } 1072 1073 static int groups_to_user(gid_t __user *dst, const struct group_info *src) 1074 { 1075 struct user_namespace *user_ns = current_user_ns(); 1076 int i; 1077 1078 for (i = 0; i < src->ngroups; i++) 1079 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i)) 1080 return -EFAULT; 1081 1082 return 0; 1083 } 1084 1085 int sock_getsockopt(struct socket *sock, int level, int optname, 1086 char __user *optval, int __user *optlen) 1087 { 1088 struct sock *sk = sock->sk; 1089 1090 union { 1091 int val; 1092 u64 val64; 1093 struct linger ling; 1094 struct timeval tm; 1095 struct sock_txtime txtime; 1096 } v; 1097 1098 int lv = sizeof(int); 1099 int len; 1100 1101 if (get_user(len, optlen)) 1102 return -EFAULT; 1103 if (len < 0) 1104 return -EINVAL; 1105 1106 memset(&v, 0, sizeof(v)); 1107 1108 switch (optname) { 1109 case SO_DEBUG: 1110 v.val = sock_flag(sk, SOCK_DBG); 1111 break; 1112 1113 case SO_DONTROUTE: 1114 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1115 break; 1116 1117 case SO_BROADCAST: 1118 v.val = sock_flag(sk, SOCK_BROADCAST); 1119 break; 1120 1121 case SO_SNDBUF: 1122 v.val = sk->sk_sndbuf; 1123 break; 1124 1125 case SO_RCVBUF: 1126 v.val = sk->sk_rcvbuf; 1127 break; 1128 1129 case SO_REUSEADDR: 1130 v.val = sk->sk_reuse; 1131 break; 1132 1133 case SO_REUSEPORT: 1134 v.val = sk->sk_reuseport; 1135 break; 1136 1137 case SO_KEEPALIVE: 1138 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1139 break; 1140 1141 case SO_TYPE: 1142 v.val = sk->sk_type; 1143 break; 1144 1145 case SO_PROTOCOL: 1146 v.val = sk->sk_protocol; 1147 break; 1148 1149 case SO_DOMAIN: 1150 v.val = sk->sk_family; 1151 break; 1152 1153 case SO_ERROR: 1154 v.val = -sock_error(sk); 1155 if (v.val == 0) 1156 v.val = xchg(&sk->sk_err_soft, 0); 1157 break; 1158 1159 case SO_OOBINLINE: 1160 v.val = sock_flag(sk, SOCK_URGINLINE); 1161 break; 1162 1163 case SO_NO_CHECK: 1164 v.val = sk->sk_no_check_tx; 1165 break; 1166 1167 case SO_PRIORITY: 1168 v.val = sk->sk_priority; 1169 break; 1170 1171 case SO_LINGER: 1172 lv = sizeof(v.ling); 1173 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1174 v.ling.l_linger = sk->sk_lingertime / HZ; 1175 break; 1176 1177 case SO_BSDCOMPAT: 1178 sock_warn_obsolete_bsdism("getsockopt"); 1179 break; 1180 1181 case SO_TIMESTAMP: 1182 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1183 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1184 break; 1185 1186 case SO_TIMESTAMPNS: 1187 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS); 1188 break; 1189 1190 case SO_TIMESTAMPING: 1191 v.val = sk->sk_tsflags; 1192 break; 1193 1194 case SO_RCVTIMEO: 1195 lv = sizeof(struct timeval); 1196 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) { 1197 v.tm.tv_sec = 0; 1198 v.tm.tv_usec = 0; 1199 } else { 1200 v.tm.tv_sec = sk->sk_rcvtimeo / HZ; 1201 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ; 1202 } 1203 break; 1204 1205 case SO_SNDTIMEO: 1206 lv = sizeof(struct timeval); 1207 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) { 1208 v.tm.tv_sec = 0; 1209 v.tm.tv_usec = 0; 1210 } else { 1211 v.tm.tv_sec = sk->sk_sndtimeo / HZ; 1212 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ; 1213 } 1214 break; 1215 1216 case SO_RCVLOWAT: 1217 v.val = sk->sk_rcvlowat; 1218 break; 1219 1220 case SO_SNDLOWAT: 1221 v.val = 1; 1222 break; 1223 1224 case SO_PASSCRED: 1225 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1226 break; 1227 1228 case SO_PEERCRED: 1229 { 1230 struct ucred peercred; 1231 if (len > sizeof(peercred)) 1232 len = sizeof(peercred); 1233 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1234 if (copy_to_user(optval, &peercred, len)) 1235 return -EFAULT; 1236 goto lenout; 1237 } 1238 1239 case SO_PEERGROUPS: 1240 { 1241 int ret, n; 1242 1243 if (!sk->sk_peer_cred) 1244 return -ENODATA; 1245 1246 n = sk->sk_peer_cred->group_info->ngroups; 1247 if (len < n * sizeof(gid_t)) { 1248 len = n * sizeof(gid_t); 1249 return put_user(len, optlen) ? -EFAULT : -ERANGE; 1250 } 1251 len = n * sizeof(gid_t); 1252 1253 ret = groups_to_user((gid_t __user *)optval, 1254 sk->sk_peer_cred->group_info); 1255 if (ret) 1256 return ret; 1257 goto lenout; 1258 } 1259 1260 case SO_PEERNAME: 1261 { 1262 char address[128]; 1263 1264 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2); 1265 if (lv < 0) 1266 return -ENOTCONN; 1267 if (lv < len) 1268 return -EINVAL; 1269 if (copy_to_user(optval, address, len)) 1270 return -EFAULT; 1271 goto lenout; 1272 } 1273 1274 /* Dubious BSD thing... Probably nobody even uses it, but 1275 * the UNIX standard wants it for whatever reason... -DaveM 1276 */ 1277 case SO_ACCEPTCONN: 1278 v.val = sk->sk_state == TCP_LISTEN; 1279 break; 1280 1281 case SO_PASSSEC: 1282 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1283 break; 1284 1285 case SO_PEERSEC: 1286 return security_socket_getpeersec_stream(sock, optval, optlen, len); 1287 1288 case SO_MARK: 1289 v.val = sk->sk_mark; 1290 break; 1291 1292 case SO_RXQ_OVFL: 1293 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1294 break; 1295 1296 case SO_WIFI_STATUS: 1297 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1298 break; 1299 1300 case SO_PEEK_OFF: 1301 if (!sock->ops->set_peek_off) 1302 return -EOPNOTSUPP; 1303 1304 v.val = sk->sk_peek_off; 1305 break; 1306 case SO_NOFCS: 1307 v.val = sock_flag(sk, SOCK_NOFCS); 1308 break; 1309 1310 case SO_BINDTODEVICE: 1311 return sock_getbindtodevice(sk, optval, optlen, len); 1312 1313 case SO_GET_FILTER: 1314 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len); 1315 if (len < 0) 1316 return len; 1317 1318 goto lenout; 1319 1320 case SO_LOCK_FILTER: 1321 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1322 break; 1323 1324 case SO_BPF_EXTENSIONS: 1325 v.val = bpf_tell_extensions(); 1326 break; 1327 1328 case SO_SELECT_ERR_QUEUE: 1329 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1330 break; 1331 1332 #ifdef CONFIG_NET_RX_BUSY_POLL 1333 case SO_BUSY_POLL: 1334 v.val = sk->sk_ll_usec; 1335 break; 1336 #endif 1337 1338 case SO_MAX_PACING_RATE: 1339 /* 32bit version */ 1340 v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U); 1341 break; 1342 1343 case SO_INCOMING_CPU: 1344 v.val = sk->sk_incoming_cpu; 1345 break; 1346 1347 case SO_MEMINFO: 1348 { 1349 u32 meminfo[SK_MEMINFO_VARS]; 1350 1351 if (get_user(len, optlen)) 1352 return -EFAULT; 1353 1354 sk_get_meminfo(sk, meminfo); 1355 1356 len = min_t(unsigned int, len, sizeof(meminfo)); 1357 if (copy_to_user(optval, &meminfo, len)) 1358 return -EFAULT; 1359 1360 goto lenout; 1361 } 1362 1363 #ifdef CONFIG_NET_RX_BUSY_POLL 1364 case SO_INCOMING_NAPI_ID: 1365 v.val = READ_ONCE(sk->sk_napi_id); 1366 1367 /* aggregate non-NAPI IDs down to 0 */ 1368 if (v.val < MIN_NAPI_ID) 1369 v.val = 0; 1370 1371 break; 1372 #endif 1373 1374 case SO_COOKIE: 1375 lv = sizeof(u64); 1376 if (len < lv) 1377 return -EINVAL; 1378 v.val64 = sock_gen_cookie(sk); 1379 break; 1380 1381 case SO_ZEROCOPY: 1382 v.val = sock_flag(sk, SOCK_ZEROCOPY); 1383 break; 1384 1385 case SO_TXTIME: 1386 lv = sizeof(v.txtime); 1387 v.txtime.clockid = sk->sk_clockid; 1388 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 1389 SOF_TXTIME_DEADLINE_MODE : 0; 1390 v.txtime.flags |= sk->sk_txtime_report_errors ? 1391 SOF_TXTIME_REPORT_ERRORS : 0; 1392 break; 1393 1394 default: 1395 /* We implement the SO_SNDLOWAT etc to not be settable 1396 * (1003.1g 7). 1397 */ 1398 return -ENOPROTOOPT; 1399 } 1400 1401 if (len > lv) 1402 len = lv; 1403 if (copy_to_user(optval, &v, len)) 1404 return -EFAULT; 1405 lenout: 1406 if (put_user(len, optlen)) 1407 return -EFAULT; 1408 return 0; 1409 } 1410 1411 /* 1412 * Initialize an sk_lock. 1413 * 1414 * (We also register the sk_lock with the lock validator.) 1415 */ 1416 static inline void sock_lock_init(struct sock *sk) 1417 { 1418 if (sk->sk_kern_sock) 1419 sock_lock_init_class_and_name( 1420 sk, 1421 af_family_kern_slock_key_strings[sk->sk_family], 1422 af_family_kern_slock_keys + sk->sk_family, 1423 af_family_kern_key_strings[sk->sk_family], 1424 af_family_kern_keys + sk->sk_family); 1425 else 1426 sock_lock_init_class_and_name( 1427 sk, 1428 af_family_slock_key_strings[sk->sk_family], 1429 af_family_slock_keys + sk->sk_family, 1430 af_family_key_strings[sk->sk_family], 1431 af_family_keys + sk->sk_family); 1432 } 1433 1434 /* 1435 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 1436 * even temporarly, because of RCU lookups. sk_node should also be left as is. 1437 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 1438 */ 1439 static void sock_copy(struct sock *nsk, const struct sock *osk) 1440 { 1441 #ifdef CONFIG_SECURITY_NETWORK 1442 void *sptr = nsk->sk_security; 1443 #endif 1444 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 1445 1446 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 1447 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 1448 1449 #ifdef CONFIG_SECURITY_NETWORK 1450 nsk->sk_security = sptr; 1451 security_sk_clone(osk, nsk); 1452 #endif 1453 } 1454 1455 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 1456 int family) 1457 { 1458 struct sock *sk; 1459 struct kmem_cache *slab; 1460 1461 slab = prot->slab; 1462 if (slab != NULL) { 1463 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 1464 if (!sk) 1465 return sk; 1466 if (priority & __GFP_ZERO) 1467 sk_prot_clear_nulls(sk, prot->obj_size); 1468 } else 1469 sk = kmalloc(prot->obj_size, priority); 1470 1471 if (sk != NULL) { 1472 if (security_sk_alloc(sk, family, priority)) 1473 goto out_free; 1474 1475 if (!try_module_get(prot->owner)) 1476 goto out_free_sec; 1477 sk_tx_queue_clear(sk); 1478 } 1479 1480 return sk; 1481 1482 out_free_sec: 1483 security_sk_free(sk); 1484 out_free: 1485 if (slab != NULL) 1486 kmem_cache_free(slab, sk); 1487 else 1488 kfree(sk); 1489 return NULL; 1490 } 1491 1492 static void sk_prot_free(struct proto *prot, struct sock *sk) 1493 { 1494 struct kmem_cache *slab; 1495 struct module *owner; 1496 1497 owner = prot->owner; 1498 slab = prot->slab; 1499 1500 cgroup_sk_free(&sk->sk_cgrp_data); 1501 mem_cgroup_sk_free(sk); 1502 security_sk_free(sk); 1503 if (slab != NULL) 1504 kmem_cache_free(slab, sk); 1505 else 1506 kfree(sk); 1507 module_put(owner); 1508 } 1509 1510 /** 1511 * sk_alloc - All socket objects are allocated here 1512 * @net: the applicable net namespace 1513 * @family: protocol family 1514 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1515 * @prot: struct proto associated with this new sock instance 1516 * @kern: is this to be a kernel socket? 1517 */ 1518 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 1519 struct proto *prot, int kern) 1520 { 1521 struct sock *sk; 1522 1523 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 1524 if (sk) { 1525 sk->sk_family = family; 1526 /* 1527 * See comment in struct sock definition to understand 1528 * why we need sk_prot_creator -acme 1529 */ 1530 sk->sk_prot = sk->sk_prot_creator = prot; 1531 sk->sk_kern_sock = kern; 1532 sock_lock_init(sk); 1533 sk->sk_net_refcnt = kern ? 0 : 1; 1534 if (likely(sk->sk_net_refcnt)) { 1535 get_net(net); 1536 sock_inuse_add(net, 1); 1537 } 1538 1539 sock_net_set(sk, net); 1540 refcount_set(&sk->sk_wmem_alloc, 1); 1541 1542 mem_cgroup_sk_alloc(sk); 1543 cgroup_sk_alloc(&sk->sk_cgrp_data); 1544 sock_update_classid(&sk->sk_cgrp_data); 1545 sock_update_netprioidx(&sk->sk_cgrp_data); 1546 } 1547 1548 return sk; 1549 } 1550 EXPORT_SYMBOL(sk_alloc); 1551 1552 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 1553 * grace period. This is the case for UDP sockets and TCP listeners. 1554 */ 1555 static void __sk_destruct(struct rcu_head *head) 1556 { 1557 struct sock *sk = container_of(head, struct sock, sk_rcu); 1558 struct sk_filter *filter; 1559 1560 if (sk->sk_destruct) 1561 sk->sk_destruct(sk); 1562 1563 filter = rcu_dereference_check(sk->sk_filter, 1564 refcount_read(&sk->sk_wmem_alloc) == 0); 1565 if (filter) { 1566 sk_filter_uncharge(sk, filter); 1567 RCU_INIT_POINTER(sk->sk_filter, NULL); 1568 } 1569 if (rcu_access_pointer(sk->sk_reuseport_cb)) 1570 reuseport_detach_sock(sk); 1571 1572 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 1573 1574 if (atomic_read(&sk->sk_omem_alloc)) 1575 pr_debug("%s: optmem leakage (%d bytes) detected\n", 1576 __func__, atomic_read(&sk->sk_omem_alloc)); 1577 1578 if (sk->sk_frag.page) { 1579 put_page(sk->sk_frag.page); 1580 sk->sk_frag.page = NULL; 1581 } 1582 1583 if (sk->sk_peer_cred) 1584 put_cred(sk->sk_peer_cred); 1585 put_pid(sk->sk_peer_pid); 1586 if (likely(sk->sk_net_refcnt)) 1587 put_net(sock_net(sk)); 1588 sk_prot_free(sk->sk_prot_creator, sk); 1589 } 1590 1591 void sk_destruct(struct sock *sk) 1592 { 1593 if (sock_flag(sk, SOCK_RCU_FREE)) 1594 call_rcu(&sk->sk_rcu, __sk_destruct); 1595 else 1596 __sk_destruct(&sk->sk_rcu); 1597 } 1598 1599 static void __sk_free(struct sock *sk) 1600 { 1601 if (likely(sk->sk_net_refcnt)) 1602 sock_inuse_add(sock_net(sk), -1); 1603 1604 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 1605 sock_diag_broadcast_destroy(sk); 1606 else 1607 sk_destruct(sk); 1608 } 1609 1610 void sk_free(struct sock *sk) 1611 { 1612 /* 1613 * We subtract one from sk_wmem_alloc and can know if 1614 * some packets are still in some tx queue. 1615 * If not null, sock_wfree() will call __sk_free(sk) later 1616 */ 1617 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 1618 __sk_free(sk); 1619 } 1620 EXPORT_SYMBOL(sk_free); 1621 1622 static void sk_init_common(struct sock *sk) 1623 { 1624 skb_queue_head_init(&sk->sk_receive_queue); 1625 skb_queue_head_init(&sk->sk_write_queue); 1626 skb_queue_head_init(&sk->sk_error_queue); 1627 1628 rwlock_init(&sk->sk_callback_lock); 1629 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 1630 af_rlock_keys + sk->sk_family, 1631 af_family_rlock_key_strings[sk->sk_family]); 1632 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 1633 af_wlock_keys + sk->sk_family, 1634 af_family_wlock_key_strings[sk->sk_family]); 1635 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 1636 af_elock_keys + sk->sk_family, 1637 af_family_elock_key_strings[sk->sk_family]); 1638 lockdep_set_class_and_name(&sk->sk_callback_lock, 1639 af_callback_keys + sk->sk_family, 1640 af_family_clock_key_strings[sk->sk_family]); 1641 } 1642 1643 /** 1644 * sk_clone_lock - clone a socket, and lock its clone 1645 * @sk: the socket to clone 1646 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1647 * 1648 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1649 */ 1650 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 1651 { 1652 struct sock *newsk; 1653 bool is_charged = true; 1654 1655 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family); 1656 if (newsk != NULL) { 1657 struct sk_filter *filter; 1658 1659 sock_copy(newsk, sk); 1660 1661 newsk->sk_prot_creator = sk->sk_prot; 1662 1663 /* SANITY */ 1664 if (likely(newsk->sk_net_refcnt)) 1665 get_net(sock_net(newsk)); 1666 sk_node_init(&newsk->sk_node); 1667 sock_lock_init(newsk); 1668 bh_lock_sock(newsk); 1669 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 1670 newsk->sk_backlog.len = 0; 1671 1672 atomic_set(&newsk->sk_rmem_alloc, 0); 1673 /* 1674 * sk_wmem_alloc set to one (see sk_free() and sock_wfree()) 1675 */ 1676 refcount_set(&newsk->sk_wmem_alloc, 1); 1677 atomic_set(&newsk->sk_omem_alloc, 0); 1678 sk_init_common(newsk); 1679 1680 newsk->sk_dst_cache = NULL; 1681 newsk->sk_dst_pending_confirm = 0; 1682 newsk->sk_wmem_queued = 0; 1683 newsk->sk_forward_alloc = 0; 1684 atomic_set(&newsk->sk_drops, 0); 1685 newsk->sk_send_head = NULL; 1686 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 1687 atomic_set(&newsk->sk_zckey, 0); 1688 1689 sock_reset_flag(newsk, SOCK_DONE); 1690 mem_cgroup_sk_alloc(newsk); 1691 cgroup_sk_alloc(&newsk->sk_cgrp_data); 1692 1693 rcu_read_lock(); 1694 filter = rcu_dereference(sk->sk_filter); 1695 if (filter != NULL) 1696 /* though it's an empty new sock, the charging may fail 1697 * if sysctl_optmem_max was changed between creation of 1698 * original socket and cloning 1699 */ 1700 is_charged = sk_filter_charge(newsk, filter); 1701 RCU_INIT_POINTER(newsk->sk_filter, filter); 1702 rcu_read_unlock(); 1703 1704 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 1705 /* We need to make sure that we don't uncharge the new 1706 * socket if we couldn't charge it in the first place 1707 * as otherwise we uncharge the parent's filter. 1708 */ 1709 if (!is_charged) 1710 RCU_INIT_POINTER(newsk->sk_filter, NULL); 1711 sk_free_unlock_clone(newsk); 1712 newsk = NULL; 1713 goto out; 1714 } 1715 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 1716 1717 newsk->sk_err = 0; 1718 newsk->sk_err_soft = 0; 1719 newsk->sk_priority = 0; 1720 newsk->sk_incoming_cpu = raw_smp_processor_id(); 1721 atomic64_set(&newsk->sk_cookie, 0); 1722 if (likely(newsk->sk_net_refcnt)) 1723 sock_inuse_add(sock_net(newsk), 1); 1724 1725 /* 1726 * Before updating sk_refcnt, we must commit prior changes to memory 1727 * (Documentation/RCU/rculist_nulls.txt for details) 1728 */ 1729 smp_wmb(); 1730 refcount_set(&newsk->sk_refcnt, 2); 1731 1732 /* 1733 * Increment the counter in the same struct proto as the master 1734 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that 1735 * is the same as sk->sk_prot->socks, as this field was copied 1736 * with memcpy). 1737 * 1738 * This _changes_ the previous behaviour, where 1739 * tcp_create_openreq_child always was incrementing the 1740 * equivalent to tcp_prot->socks (inet_sock_nr), so this have 1741 * to be taken into account in all callers. -acme 1742 */ 1743 sk_refcnt_debug_inc(newsk); 1744 sk_set_socket(newsk, NULL); 1745 newsk->sk_wq = NULL; 1746 1747 if (newsk->sk_prot->sockets_allocated) 1748 sk_sockets_allocated_inc(newsk); 1749 1750 if (sock_needs_netstamp(sk) && 1751 newsk->sk_flags & SK_FLAGS_TIMESTAMP) 1752 net_enable_timestamp(); 1753 } 1754 out: 1755 return newsk; 1756 } 1757 EXPORT_SYMBOL_GPL(sk_clone_lock); 1758 1759 void sk_free_unlock_clone(struct sock *sk) 1760 { 1761 /* It is still raw copy of parent, so invalidate 1762 * destructor and make plain sk_free() */ 1763 sk->sk_destruct = NULL; 1764 bh_unlock_sock(sk); 1765 sk_free(sk); 1766 } 1767 EXPORT_SYMBOL_GPL(sk_free_unlock_clone); 1768 1769 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 1770 { 1771 u32 max_segs = 1; 1772 1773 sk_dst_set(sk, dst); 1774 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps; 1775 if (sk->sk_route_caps & NETIF_F_GSO) 1776 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 1777 sk->sk_route_caps &= ~sk->sk_route_nocaps; 1778 if (sk_can_gso(sk)) { 1779 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 1780 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 1781 } else { 1782 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 1783 sk->sk_gso_max_size = dst->dev->gso_max_size; 1784 max_segs = max_t(u32, dst->dev->gso_max_segs, 1); 1785 } 1786 } 1787 sk->sk_gso_max_segs = max_segs; 1788 } 1789 EXPORT_SYMBOL_GPL(sk_setup_caps); 1790 1791 /* 1792 * Simple resource managers for sockets. 1793 */ 1794 1795 1796 /* 1797 * Write buffer destructor automatically called from kfree_skb. 1798 */ 1799 void sock_wfree(struct sk_buff *skb) 1800 { 1801 struct sock *sk = skb->sk; 1802 unsigned int len = skb->truesize; 1803 1804 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 1805 /* 1806 * Keep a reference on sk_wmem_alloc, this will be released 1807 * after sk_write_space() call 1808 */ 1809 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 1810 sk->sk_write_space(sk); 1811 len = 1; 1812 } 1813 /* 1814 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 1815 * could not do because of in-flight packets 1816 */ 1817 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 1818 __sk_free(sk); 1819 } 1820 EXPORT_SYMBOL(sock_wfree); 1821 1822 /* This variant of sock_wfree() is used by TCP, 1823 * since it sets SOCK_USE_WRITE_QUEUE. 1824 */ 1825 void __sock_wfree(struct sk_buff *skb) 1826 { 1827 struct sock *sk = skb->sk; 1828 1829 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 1830 __sk_free(sk); 1831 } 1832 1833 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 1834 { 1835 skb_orphan(skb); 1836 skb->sk = sk; 1837 #ifdef CONFIG_INET 1838 if (unlikely(!sk_fullsock(sk))) { 1839 skb->destructor = sock_edemux; 1840 sock_hold(sk); 1841 return; 1842 } 1843 #endif 1844 skb->destructor = sock_wfree; 1845 skb_set_hash_from_sk(skb, sk); 1846 /* 1847 * We used to take a refcount on sk, but following operation 1848 * is enough to guarantee sk_free() wont free this sock until 1849 * all in-flight packets are completed 1850 */ 1851 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 1852 } 1853 EXPORT_SYMBOL(skb_set_owner_w); 1854 1855 /* This helper is used by netem, as it can hold packets in its 1856 * delay queue. We want to allow the owner socket to send more 1857 * packets, as if they were already TX completed by a typical driver. 1858 * But we also want to keep skb->sk set because some packet schedulers 1859 * rely on it (sch_fq for example). 1860 */ 1861 void skb_orphan_partial(struct sk_buff *skb) 1862 { 1863 if (skb_is_tcp_pure_ack(skb)) 1864 return; 1865 1866 if (skb->destructor == sock_wfree 1867 #ifdef CONFIG_INET 1868 || skb->destructor == tcp_wfree 1869 #endif 1870 ) { 1871 struct sock *sk = skb->sk; 1872 1873 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 1874 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)); 1875 skb->destructor = sock_efree; 1876 } 1877 } else { 1878 skb_orphan(skb); 1879 } 1880 } 1881 EXPORT_SYMBOL(skb_orphan_partial); 1882 1883 /* 1884 * Read buffer destructor automatically called from kfree_skb. 1885 */ 1886 void sock_rfree(struct sk_buff *skb) 1887 { 1888 struct sock *sk = skb->sk; 1889 unsigned int len = skb->truesize; 1890 1891 atomic_sub(len, &sk->sk_rmem_alloc); 1892 sk_mem_uncharge(sk, len); 1893 } 1894 EXPORT_SYMBOL(sock_rfree); 1895 1896 /* 1897 * Buffer destructor for skbs that are not used directly in read or write 1898 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 1899 */ 1900 void sock_efree(struct sk_buff *skb) 1901 { 1902 sock_put(skb->sk); 1903 } 1904 EXPORT_SYMBOL(sock_efree); 1905 1906 kuid_t sock_i_uid(struct sock *sk) 1907 { 1908 kuid_t uid; 1909 1910 read_lock_bh(&sk->sk_callback_lock); 1911 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 1912 read_unlock_bh(&sk->sk_callback_lock); 1913 return uid; 1914 } 1915 EXPORT_SYMBOL(sock_i_uid); 1916 1917 unsigned long sock_i_ino(struct sock *sk) 1918 { 1919 unsigned long ino; 1920 1921 read_lock_bh(&sk->sk_callback_lock); 1922 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 1923 read_unlock_bh(&sk->sk_callback_lock); 1924 return ino; 1925 } 1926 EXPORT_SYMBOL(sock_i_ino); 1927 1928 /* 1929 * Allocate a skb from the socket's send buffer. 1930 */ 1931 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 1932 gfp_t priority) 1933 { 1934 if (force || refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) { 1935 struct sk_buff *skb = alloc_skb(size, priority); 1936 if (skb) { 1937 skb_set_owner_w(skb, sk); 1938 return skb; 1939 } 1940 } 1941 return NULL; 1942 } 1943 EXPORT_SYMBOL(sock_wmalloc); 1944 1945 static void sock_ofree(struct sk_buff *skb) 1946 { 1947 struct sock *sk = skb->sk; 1948 1949 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 1950 } 1951 1952 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 1953 gfp_t priority) 1954 { 1955 struct sk_buff *skb; 1956 1957 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 1958 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 1959 sysctl_optmem_max) 1960 return NULL; 1961 1962 skb = alloc_skb(size, priority); 1963 if (!skb) 1964 return NULL; 1965 1966 atomic_add(skb->truesize, &sk->sk_omem_alloc); 1967 skb->sk = sk; 1968 skb->destructor = sock_ofree; 1969 return skb; 1970 } 1971 1972 /* 1973 * Allocate a memory block from the socket's option memory buffer. 1974 */ 1975 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 1976 { 1977 if ((unsigned int)size <= sysctl_optmem_max && 1978 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { 1979 void *mem; 1980 /* First do the add, to avoid the race if kmalloc 1981 * might sleep. 1982 */ 1983 atomic_add(size, &sk->sk_omem_alloc); 1984 mem = kmalloc(size, priority); 1985 if (mem) 1986 return mem; 1987 atomic_sub(size, &sk->sk_omem_alloc); 1988 } 1989 return NULL; 1990 } 1991 EXPORT_SYMBOL(sock_kmalloc); 1992 1993 /* Free an option memory block. Note, we actually want the inline 1994 * here as this allows gcc to detect the nullify and fold away the 1995 * condition entirely. 1996 */ 1997 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 1998 const bool nullify) 1999 { 2000 if (WARN_ON_ONCE(!mem)) 2001 return; 2002 if (nullify) 2003 kzfree(mem); 2004 else 2005 kfree(mem); 2006 atomic_sub(size, &sk->sk_omem_alloc); 2007 } 2008 2009 void sock_kfree_s(struct sock *sk, void *mem, int size) 2010 { 2011 __sock_kfree_s(sk, mem, size, false); 2012 } 2013 EXPORT_SYMBOL(sock_kfree_s); 2014 2015 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2016 { 2017 __sock_kfree_s(sk, mem, size, true); 2018 } 2019 EXPORT_SYMBOL(sock_kzfree_s); 2020 2021 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2022 I think, these locks should be removed for datagram sockets. 2023 */ 2024 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2025 { 2026 DEFINE_WAIT(wait); 2027 2028 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2029 for (;;) { 2030 if (!timeo) 2031 break; 2032 if (signal_pending(current)) 2033 break; 2034 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2035 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2036 if (refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) 2037 break; 2038 if (sk->sk_shutdown & SEND_SHUTDOWN) 2039 break; 2040 if (sk->sk_err) 2041 break; 2042 timeo = schedule_timeout(timeo); 2043 } 2044 finish_wait(sk_sleep(sk), &wait); 2045 return timeo; 2046 } 2047 2048 2049 /* 2050 * Generic send/receive buffer handlers 2051 */ 2052 2053 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2054 unsigned long data_len, int noblock, 2055 int *errcode, int max_page_order) 2056 { 2057 struct sk_buff *skb; 2058 long timeo; 2059 int err; 2060 2061 timeo = sock_sndtimeo(sk, noblock); 2062 for (;;) { 2063 err = sock_error(sk); 2064 if (err != 0) 2065 goto failure; 2066 2067 err = -EPIPE; 2068 if (sk->sk_shutdown & SEND_SHUTDOWN) 2069 goto failure; 2070 2071 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf) 2072 break; 2073 2074 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2075 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2076 err = -EAGAIN; 2077 if (!timeo) 2078 goto failure; 2079 if (signal_pending(current)) 2080 goto interrupted; 2081 timeo = sock_wait_for_wmem(sk, timeo); 2082 } 2083 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 2084 errcode, sk->sk_allocation); 2085 if (skb) 2086 skb_set_owner_w(skb, sk); 2087 return skb; 2088 2089 interrupted: 2090 err = sock_intr_errno(timeo); 2091 failure: 2092 *errcode = err; 2093 return NULL; 2094 } 2095 EXPORT_SYMBOL(sock_alloc_send_pskb); 2096 2097 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size, 2098 int noblock, int *errcode) 2099 { 2100 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0); 2101 } 2102 EXPORT_SYMBOL(sock_alloc_send_skb); 2103 2104 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, 2105 struct sockcm_cookie *sockc) 2106 { 2107 u32 tsflags; 2108 2109 switch (cmsg->cmsg_type) { 2110 case SO_MARK: 2111 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 2112 return -EPERM; 2113 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2114 return -EINVAL; 2115 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 2116 break; 2117 case SO_TIMESTAMPING: 2118 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2119 return -EINVAL; 2120 2121 tsflags = *(u32 *)CMSG_DATA(cmsg); 2122 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 2123 return -EINVAL; 2124 2125 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 2126 sockc->tsflags |= tsflags; 2127 break; 2128 case SCM_TXTIME: 2129 if (!sock_flag(sk, SOCK_TXTIME)) 2130 return -EINVAL; 2131 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 2132 return -EINVAL; 2133 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); 2134 break; 2135 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 2136 case SCM_RIGHTS: 2137 case SCM_CREDENTIALS: 2138 break; 2139 default: 2140 return -EINVAL; 2141 } 2142 return 0; 2143 } 2144 EXPORT_SYMBOL(__sock_cmsg_send); 2145 2146 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 2147 struct sockcm_cookie *sockc) 2148 { 2149 struct cmsghdr *cmsg; 2150 int ret; 2151 2152 for_each_cmsghdr(cmsg, msg) { 2153 if (!CMSG_OK(msg, cmsg)) 2154 return -EINVAL; 2155 if (cmsg->cmsg_level != SOL_SOCKET) 2156 continue; 2157 ret = __sock_cmsg_send(sk, msg, cmsg, sockc); 2158 if (ret) 2159 return ret; 2160 } 2161 return 0; 2162 } 2163 EXPORT_SYMBOL(sock_cmsg_send); 2164 2165 static void sk_enter_memory_pressure(struct sock *sk) 2166 { 2167 if (!sk->sk_prot->enter_memory_pressure) 2168 return; 2169 2170 sk->sk_prot->enter_memory_pressure(sk); 2171 } 2172 2173 static void sk_leave_memory_pressure(struct sock *sk) 2174 { 2175 if (sk->sk_prot->leave_memory_pressure) { 2176 sk->sk_prot->leave_memory_pressure(sk); 2177 } else { 2178 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 2179 2180 if (memory_pressure && *memory_pressure) 2181 *memory_pressure = 0; 2182 } 2183 } 2184 2185 /* On 32bit arches, an skb frag is limited to 2^15 */ 2186 #define SKB_FRAG_PAGE_ORDER get_order(32768) 2187 2188 /** 2189 * skb_page_frag_refill - check that a page_frag contains enough room 2190 * @sz: minimum size of the fragment we want to get 2191 * @pfrag: pointer to page_frag 2192 * @gfp: priority for memory allocation 2193 * 2194 * Note: While this allocator tries to use high order pages, there is 2195 * no guarantee that allocations succeed. Therefore, @sz MUST be 2196 * less or equal than PAGE_SIZE. 2197 */ 2198 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 2199 { 2200 if (pfrag->page) { 2201 if (page_ref_count(pfrag->page) == 1) { 2202 pfrag->offset = 0; 2203 return true; 2204 } 2205 if (pfrag->offset + sz <= pfrag->size) 2206 return true; 2207 put_page(pfrag->page); 2208 } 2209 2210 pfrag->offset = 0; 2211 if (SKB_FRAG_PAGE_ORDER) { 2212 /* Avoid direct reclaim but allow kswapd to wake */ 2213 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 2214 __GFP_COMP | __GFP_NOWARN | 2215 __GFP_NORETRY, 2216 SKB_FRAG_PAGE_ORDER); 2217 if (likely(pfrag->page)) { 2218 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 2219 return true; 2220 } 2221 } 2222 pfrag->page = alloc_page(gfp); 2223 if (likely(pfrag->page)) { 2224 pfrag->size = PAGE_SIZE; 2225 return true; 2226 } 2227 return false; 2228 } 2229 EXPORT_SYMBOL(skb_page_frag_refill); 2230 2231 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2232 { 2233 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2234 return true; 2235 2236 sk_enter_memory_pressure(sk); 2237 sk_stream_moderate_sndbuf(sk); 2238 return false; 2239 } 2240 EXPORT_SYMBOL(sk_page_frag_refill); 2241 2242 int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg, 2243 int sg_start, int *sg_curr_index, unsigned int *sg_curr_size, 2244 int first_coalesce) 2245 { 2246 int sg_curr = *sg_curr_index, use = 0, rc = 0; 2247 unsigned int size = *sg_curr_size; 2248 struct page_frag *pfrag; 2249 struct scatterlist *sge; 2250 2251 len -= size; 2252 pfrag = sk_page_frag(sk); 2253 2254 while (len > 0) { 2255 unsigned int orig_offset; 2256 2257 if (!sk_page_frag_refill(sk, pfrag)) { 2258 rc = -ENOMEM; 2259 goto out; 2260 } 2261 2262 use = min_t(int, len, pfrag->size - pfrag->offset); 2263 2264 if (!sk_wmem_schedule(sk, use)) { 2265 rc = -ENOMEM; 2266 goto out; 2267 } 2268 2269 sk_mem_charge(sk, use); 2270 size += use; 2271 orig_offset = pfrag->offset; 2272 pfrag->offset += use; 2273 2274 sge = sg + sg_curr - 1; 2275 if (sg_curr > first_coalesce && sg_page(sge) == pfrag->page && 2276 sge->offset + sge->length == orig_offset) { 2277 sge->length += use; 2278 } else { 2279 sge = sg + sg_curr; 2280 sg_unmark_end(sge); 2281 sg_set_page(sge, pfrag->page, use, orig_offset); 2282 get_page(pfrag->page); 2283 sg_curr++; 2284 2285 if (sg_curr == MAX_SKB_FRAGS) 2286 sg_curr = 0; 2287 2288 if (sg_curr == sg_start) { 2289 rc = -ENOSPC; 2290 break; 2291 } 2292 } 2293 2294 len -= use; 2295 } 2296 out: 2297 *sg_curr_size = size; 2298 *sg_curr_index = sg_curr; 2299 return rc; 2300 } 2301 EXPORT_SYMBOL(sk_alloc_sg); 2302 2303 static void __lock_sock(struct sock *sk) 2304 __releases(&sk->sk_lock.slock) 2305 __acquires(&sk->sk_lock.slock) 2306 { 2307 DEFINE_WAIT(wait); 2308 2309 for (;;) { 2310 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2311 TASK_UNINTERRUPTIBLE); 2312 spin_unlock_bh(&sk->sk_lock.slock); 2313 schedule(); 2314 spin_lock_bh(&sk->sk_lock.slock); 2315 if (!sock_owned_by_user(sk)) 2316 break; 2317 } 2318 finish_wait(&sk->sk_lock.wq, &wait); 2319 } 2320 2321 void __release_sock(struct sock *sk) 2322 __releases(&sk->sk_lock.slock) 2323 __acquires(&sk->sk_lock.slock) 2324 { 2325 struct sk_buff *skb, *next; 2326 2327 while ((skb = sk->sk_backlog.head) != NULL) { 2328 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2329 2330 spin_unlock_bh(&sk->sk_lock.slock); 2331 2332 do { 2333 next = skb->next; 2334 prefetch(next); 2335 WARN_ON_ONCE(skb_dst_is_noref(skb)); 2336 skb_mark_not_on_list(skb); 2337 sk_backlog_rcv(sk, skb); 2338 2339 cond_resched(); 2340 2341 skb = next; 2342 } while (skb != NULL); 2343 2344 spin_lock_bh(&sk->sk_lock.slock); 2345 } 2346 2347 /* 2348 * Doing the zeroing here guarantee we can not loop forever 2349 * while a wild producer attempts to flood us. 2350 */ 2351 sk->sk_backlog.len = 0; 2352 } 2353 2354 void __sk_flush_backlog(struct sock *sk) 2355 { 2356 spin_lock_bh(&sk->sk_lock.slock); 2357 __release_sock(sk); 2358 spin_unlock_bh(&sk->sk_lock.slock); 2359 } 2360 2361 /** 2362 * sk_wait_data - wait for data to arrive at sk_receive_queue 2363 * @sk: sock to wait on 2364 * @timeo: for how long 2365 * @skb: last skb seen on sk_receive_queue 2366 * 2367 * Now socket state including sk->sk_err is changed only under lock, 2368 * hence we may omit checks after joining wait queue. 2369 * We check receive queue before schedule() only as optimization; 2370 * it is very likely that release_sock() added new data. 2371 */ 2372 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 2373 { 2374 DEFINE_WAIT_FUNC(wait, woken_wake_function); 2375 int rc; 2376 2377 add_wait_queue(sk_sleep(sk), &wait); 2378 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2379 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 2380 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2381 remove_wait_queue(sk_sleep(sk), &wait); 2382 return rc; 2383 } 2384 EXPORT_SYMBOL(sk_wait_data); 2385 2386 /** 2387 * __sk_mem_raise_allocated - increase memory_allocated 2388 * @sk: socket 2389 * @size: memory size to allocate 2390 * @amt: pages to allocate 2391 * @kind: allocation type 2392 * 2393 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc 2394 */ 2395 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 2396 { 2397 struct proto *prot = sk->sk_prot; 2398 long allocated = sk_memory_allocated_add(sk, amt); 2399 bool charged = true; 2400 2401 if (mem_cgroup_sockets_enabled && sk->sk_memcg && 2402 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt))) 2403 goto suppress_allocation; 2404 2405 /* Under limit. */ 2406 if (allocated <= sk_prot_mem_limits(sk, 0)) { 2407 sk_leave_memory_pressure(sk); 2408 return 1; 2409 } 2410 2411 /* Under pressure. */ 2412 if (allocated > sk_prot_mem_limits(sk, 1)) 2413 sk_enter_memory_pressure(sk); 2414 2415 /* Over hard limit. */ 2416 if (allocated > sk_prot_mem_limits(sk, 2)) 2417 goto suppress_allocation; 2418 2419 /* guarantee minimum buffer size under pressure */ 2420 if (kind == SK_MEM_RECV) { 2421 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 2422 return 1; 2423 2424 } else { /* SK_MEM_SEND */ 2425 int wmem0 = sk_get_wmem0(sk, prot); 2426 2427 if (sk->sk_type == SOCK_STREAM) { 2428 if (sk->sk_wmem_queued < wmem0) 2429 return 1; 2430 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 2431 return 1; 2432 } 2433 } 2434 2435 if (sk_has_memory_pressure(sk)) { 2436 int alloc; 2437 2438 if (!sk_under_memory_pressure(sk)) 2439 return 1; 2440 alloc = sk_sockets_allocated_read_positive(sk); 2441 if (sk_prot_mem_limits(sk, 2) > alloc * 2442 sk_mem_pages(sk->sk_wmem_queued + 2443 atomic_read(&sk->sk_rmem_alloc) + 2444 sk->sk_forward_alloc)) 2445 return 1; 2446 } 2447 2448 suppress_allocation: 2449 2450 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 2451 sk_stream_moderate_sndbuf(sk); 2452 2453 /* Fail only if socket is _under_ its sndbuf. 2454 * In this case we cannot block, so that we have to fail. 2455 */ 2456 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) 2457 return 1; 2458 } 2459 2460 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged)) 2461 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 2462 2463 sk_memory_allocated_sub(sk, amt); 2464 2465 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2466 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt); 2467 2468 return 0; 2469 } 2470 EXPORT_SYMBOL(__sk_mem_raise_allocated); 2471 2472 /** 2473 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 2474 * @sk: socket 2475 * @size: memory size to allocate 2476 * @kind: allocation type 2477 * 2478 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 2479 * rmem allocation. This function assumes that protocols which have 2480 * memory_pressure use sk_wmem_queued as write buffer accounting. 2481 */ 2482 int __sk_mem_schedule(struct sock *sk, int size, int kind) 2483 { 2484 int ret, amt = sk_mem_pages(size); 2485 2486 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT; 2487 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 2488 if (!ret) 2489 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT; 2490 return ret; 2491 } 2492 EXPORT_SYMBOL(__sk_mem_schedule); 2493 2494 /** 2495 * __sk_mem_reduce_allocated - reclaim memory_allocated 2496 * @sk: socket 2497 * @amount: number of quanta 2498 * 2499 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 2500 */ 2501 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 2502 { 2503 sk_memory_allocated_sub(sk, amount); 2504 2505 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2506 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 2507 2508 if (sk_under_memory_pressure(sk) && 2509 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 2510 sk_leave_memory_pressure(sk); 2511 } 2512 EXPORT_SYMBOL(__sk_mem_reduce_allocated); 2513 2514 /** 2515 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 2516 * @sk: socket 2517 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple) 2518 */ 2519 void __sk_mem_reclaim(struct sock *sk, int amount) 2520 { 2521 amount >>= SK_MEM_QUANTUM_SHIFT; 2522 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT; 2523 __sk_mem_reduce_allocated(sk, amount); 2524 } 2525 EXPORT_SYMBOL(__sk_mem_reclaim); 2526 2527 int sk_set_peek_off(struct sock *sk, int val) 2528 { 2529 sk->sk_peek_off = val; 2530 return 0; 2531 } 2532 EXPORT_SYMBOL_GPL(sk_set_peek_off); 2533 2534 /* 2535 * Set of default routines for initialising struct proto_ops when 2536 * the protocol does not support a particular function. In certain 2537 * cases where it makes no sense for a protocol to have a "do nothing" 2538 * function, some default processing is provided. 2539 */ 2540 2541 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 2542 { 2543 return -EOPNOTSUPP; 2544 } 2545 EXPORT_SYMBOL(sock_no_bind); 2546 2547 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 2548 int len, int flags) 2549 { 2550 return -EOPNOTSUPP; 2551 } 2552 EXPORT_SYMBOL(sock_no_connect); 2553 2554 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 2555 { 2556 return -EOPNOTSUPP; 2557 } 2558 EXPORT_SYMBOL(sock_no_socketpair); 2559 2560 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, 2561 bool kern) 2562 { 2563 return -EOPNOTSUPP; 2564 } 2565 EXPORT_SYMBOL(sock_no_accept); 2566 2567 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 2568 int peer) 2569 { 2570 return -EOPNOTSUPP; 2571 } 2572 EXPORT_SYMBOL(sock_no_getname); 2573 2574 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 2575 { 2576 return -EOPNOTSUPP; 2577 } 2578 EXPORT_SYMBOL(sock_no_ioctl); 2579 2580 int sock_no_listen(struct socket *sock, int backlog) 2581 { 2582 return -EOPNOTSUPP; 2583 } 2584 EXPORT_SYMBOL(sock_no_listen); 2585 2586 int sock_no_shutdown(struct socket *sock, int how) 2587 { 2588 return -EOPNOTSUPP; 2589 } 2590 EXPORT_SYMBOL(sock_no_shutdown); 2591 2592 int sock_no_setsockopt(struct socket *sock, int level, int optname, 2593 char __user *optval, unsigned int optlen) 2594 { 2595 return -EOPNOTSUPP; 2596 } 2597 EXPORT_SYMBOL(sock_no_setsockopt); 2598 2599 int sock_no_getsockopt(struct socket *sock, int level, int optname, 2600 char __user *optval, int __user *optlen) 2601 { 2602 return -EOPNOTSUPP; 2603 } 2604 EXPORT_SYMBOL(sock_no_getsockopt); 2605 2606 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 2607 { 2608 return -EOPNOTSUPP; 2609 } 2610 EXPORT_SYMBOL(sock_no_sendmsg); 2611 2612 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 2613 { 2614 return -EOPNOTSUPP; 2615 } 2616 EXPORT_SYMBOL(sock_no_sendmsg_locked); 2617 2618 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 2619 int flags) 2620 { 2621 return -EOPNOTSUPP; 2622 } 2623 EXPORT_SYMBOL(sock_no_recvmsg); 2624 2625 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 2626 { 2627 /* Mirror missing mmap method error code */ 2628 return -ENODEV; 2629 } 2630 EXPORT_SYMBOL(sock_no_mmap); 2631 2632 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) 2633 { 2634 ssize_t res; 2635 struct msghdr msg = {.msg_flags = flags}; 2636 struct kvec iov; 2637 char *kaddr = kmap(page); 2638 iov.iov_base = kaddr + offset; 2639 iov.iov_len = size; 2640 res = kernel_sendmsg(sock, &msg, &iov, 1, size); 2641 kunmap(page); 2642 return res; 2643 } 2644 EXPORT_SYMBOL(sock_no_sendpage); 2645 2646 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page, 2647 int offset, size_t size, int flags) 2648 { 2649 ssize_t res; 2650 struct msghdr msg = {.msg_flags = flags}; 2651 struct kvec iov; 2652 char *kaddr = kmap(page); 2653 2654 iov.iov_base = kaddr + offset; 2655 iov.iov_len = size; 2656 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size); 2657 kunmap(page); 2658 return res; 2659 } 2660 EXPORT_SYMBOL(sock_no_sendpage_locked); 2661 2662 /* 2663 * Default Socket Callbacks 2664 */ 2665 2666 static void sock_def_wakeup(struct sock *sk) 2667 { 2668 struct socket_wq *wq; 2669 2670 rcu_read_lock(); 2671 wq = rcu_dereference(sk->sk_wq); 2672 if (skwq_has_sleeper(wq)) 2673 wake_up_interruptible_all(&wq->wait); 2674 rcu_read_unlock(); 2675 } 2676 2677 static void sock_def_error_report(struct sock *sk) 2678 { 2679 struct socket_wq *wq; 2680 2681 rcu_read_lock(); 2682 wq = rcu_dereference(sk->sk_wq); 2683 if (skwq_has_sleeper(wq)) 2684 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 2685 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 2686 rcu_read_unlock(); 2687 } 2688 2689 static void sock_def_readable(struct sock *sk) 2690 { 2691 struct socket_wq *wq; 2692 2693 rcu_read_lock(); 2694 wq = rcu_dereference(sk->sk_wq); 2695 if (skwq_has_sleeper(wq)) 2696 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 2697 EPOLLRDNORM | EPOLLRDBAND); 2698 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 2699 rcu_read_unlock(); 2700 } 2701 2702 static void sock_def_write_space(struct sock *sk) 2703 { 2704 struct socket_wq *wq; 2705 2706 rcu_read_lock(); 2707 2708 /* Do not wake up a writer until he can make "significant" 2709 * progress. --DaveM 2710 */ 2711 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) { 2712 wq = rcu_dereference(sk->sk_wq); 2713 if (skwq_has_sleeper(wq)) 2714 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 2715 EPOLLWRNORM | EPOLLWRBAND); 2716 2717 /* Should agree with poll, otherwise some programs break */ 2718 if (sock_writeable(sk)) 2719 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 2720 } 2721 2722 rcu_read_unlock(); 2723 } 2724 2725 static void sock_def_destruct(struct sock *sk) 2726 { 2727 } 2728 2729 void sk_send_sigurg(struct sock *sk) 2730 { 2731 if (sk->sk_socket && sk->sk_socket->file) 2732 if (send_sigurg(&sk->sk_socket->file->f_owner)) 2733 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 2734 } 2735 EXPORT_SYMBOL(sk_send_sigurg); 2736 2737 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 2738 unsigned long expires) 2739 { 2740 if (!mod_timer(timer, expires)) 2741 sock_hold(sk); 2742 } 2743 EXPORT_SYMBOL(sk_reset_timer); 2744 2745 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 2746 { 2747 if (del_timer(timer)) 2748 __sock_put(sk); 2749 } 2750 EXPORT_SYMBOL(sk_stop_timer); 2751 2752 void sock_init_data(struct socket *sock, struct sock *sk) 2753 { 2754 sk_init_common(sk); 2755 sk->sk_send_head = NULL; 2756 2757 timer_setup(&sk->sk_timer, NULL, 0); 2758 2759 sk->sk_allocation = GFP_KERNEL; 2760 sk->sk_rcvbuf = sysctl_rmem_default; 2761 sk->sk_sndbuf = sysctl_wmem_default; 2762 sk->sk_state = TCP_CLOSE; 2763 sk_set_socket(sk, sock); 2764 2765 sock_set_flag(sk, SOCK_ZAPPED); 2766 2767 if (sock) { 2768 sk->sk_type = sock->type; 2769 sk->sk_wq = sock->wq; 2770 sock->sk = sk; 2771 sk->sk_uid = SOCK_INODE(sock)->i_uid; 2772 } else { 2773 sk->sk_wq = NULL; 2774 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0); 2775 } 2776 2777 rwlock_init(&sk->sk_callback_lock); 2778 if (sk->sk_kern_sock) 2779 lockdep_set_class_and_name( 2780 &sk->sk_callback_lock, 2781 af_kern_callback_keys + sk->sk_family, 2782 af_family_kern_clock_key_strings[sk->sk_family]); 2783 else 2784 lockdep_set_class_and_name( 2785 &sk->sk_callback_lock, 2786 af_callback_keys + sk->sk_family, 2787 af_family_clock_key_strings[sk->sk_family]); 2788 2789 sk->sk_state_change = sock_def_wakeup; 2790 sk->sk_data_ready = sock_def_readable; 2791 sk->sk_write_space = sock_def_write_space; 2792 sk->sk_error_report = sock_def_error_report; 2793 sk->sk_destruct = sock_def_destruct; 2794 2795 sk->sk_frag.page = NULL; 2796 sk->sk_frag.offset = 0; 2797 sk->sk_peek_off = -1; 2798 2799 sk->sk_peer_pid = NULL; 2800 sk->sk_peer_cred = NULL; 2801 sk->sk_write_pending = 0; 2802 sk->sk_rcvlowat = 1; 2803 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 2804 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 2805 2806 sk->sk_stamp = SK_DEFAULT_STAMP; 2807 atomic_set(&sk->sk_zckey, 0); 2808 2809 #ifdef CONFIG_NET_RX_BUSY_POLL 2810 sk->sk_napi_id = 0; 2811 sk->sk_ll_usec = sysctl_net_busy_read; 2812 #endif 2813 2814 sk->sk_max_pacing_rate = ~0UL; 2815 sk->sk_pacing_rate = ~0UL; 2816 sk->sk_pacing_shift = 10; 2817 sk->sk_incoming_cpu = -1; 2818 2819 sk_rx_queue_clear(sk); 2820 /* 2821 * Before updating sk_refcnt, we must commit prior changes to memory 2822 * (Documentation/RCU/rculist_nulls.txt for details) 2823 */ 2824 smp_wmb(); 2825 refcount_set(&sk->sk_refcnt, 1); 2826 atomic_set(&sk->sk_drops, 0); 2827 } 2828 EXPORT_SYMBOL(sock_init_data); 2829 2830 void lock_sock_nested(struct sock *sk, int subclass) 2831 { 2832 might_sleep(); 2833 spin_lock_bh(&sk->sk_lock.slock); 2834 if (sk->sk_lock.owned) 2835 __lock_sock(sk); 2836 sk->sk_lock.owned = 1; 2837 spin_unlock(&sk->sk_lock.slock); 2838 /* 2839 * The sk_lock has mutex_lock() semantics here: 2840 */ 2841 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 2842 local_bh_enable(); 2843 } 2844 EXPORT_SYMBOL(lock_sock_nested); 2845 2846 void release_sock(struct sock *sk) 2847 { 2848 spin_lock_bh(&sk->sk_lock.slock); 2849 if (sk->sk_backlog.tail) 2850 __release_sock(sk); 2851 2852 /* Warning : release_cb() might need to release sk ownership, 2853 * ie call sock_release_ownership(sk) before us. 2854 */ 2855 if (sk->sk_prot->release_cb) 2856 sk->sk_prot->release_cb(sk); 2857 2858 sock_release_ownership(sk); 2859 if (waitqueue_active(&sk->sk_lock.wq)) 2860 wake_up(&sk->sk_lock.wq); 2861 spin_unlock_bh(&sk->sk_lock.slock); 2862 } 2863 EXPORT_SYMBOL(release_sock); 2864 2865 /** 2866 * lock_sock_fast - fast version of lock_sock 2867 * @sk: socket 2868 * 2869 * This version should be used for very small section, where process wont block 2870 * return false if fast path is taken: 2871 * 2872 * sk_lock.slock locked, owned = 0, BH disabled 2873 * 2874 * return true if slow path is taken: 2875 * 2876 * sk_lock.slock unlocked, owned = 1, BH enabled 2877 */ 2878 bool lock_sock_fast(struct sock *sk) 2879 { 2880 might_sleep(); 2881 spin_lock_bh(&sk->sk_lock.slock); 2882 2883 if (!sk->sk_lock.owned) 2884 /* 2885 * Note : We must disable BH 2886 */ 2887 return false; 2888 2889 __lock_sock(sk); 2890 sk->sk_lock.owned = 1; 2891 spin_unlock(&sk->sk_lock.slock); 2892 /* 2893 * The sk_lock has mutex_lock() semantics here: 2894 */ 2895 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_); 2896 local_bh_enable(); 2897 return true; 2898 } 2899 EXPORT_SYMBOL(lock_sock_fast); 2900 2901 int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp) 2902 { 2903 struct timeval tv; 2904 2905 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2906 tv = ktime_to_timeval(sk->sk_stamp); 2907 if (tv.tv_sec == -1) 2908 return -ENOENT; 2909 if (tv.tv_sec == 0) { 2910 sk->sk_stamp = ktime_get_real(); 2911 tv = ktime_to_timeval(sk->sk_stamp); 2912 } 2913 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0; 2914 } 2915 EXPORT_SYMBOL(sock_get_timestamp); 2916 2917 int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp) 2918 { 2919 struct timespec ts; 2920 2921 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2922 ts = ktime_to_timespec(sk->sk_stamp); 2923 if (ts.tv_sec == -1) 2924 return -ENOENT; 2925 if (ts.tv_sec == 0) { 2926 sk->sk_stamp = ktime_get_real(); 2927 ts = ktime_to_timespec(sk->sk_stamp); 2928 } 2929 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0; 2930 } 2931 EXPORT_SYMBOL(sock_get_timestampns); 2932 2933 void sock_enable_timestamp(struct sock *sk, int flag) 2934 { 2935 if (!sock_flag(sk, flag)) { 2936 unsigned long previous_flags = sk->sk_flags; 2937 2938 sock_set_flag(sk, flag); 2939 /* 2940 * we just set one of the two flags which require net 2941 * time stamping, but time stamping might have been on 2942 * already because of the other one 2943 */ 2944 if (sock_needs_netstamp(sk) && 2945 !(previous_flags & SK_FLAGS_TIMESTAMP)) 2946 net_enable_timestamp(); 2947 } 2948 } 2949 2950 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 2951 int level, int type) 2952 { 2953 struct sock_exterr_skb *serr; 2954 struct sk_buff *skb; 2955 int copied, err; 2956 2957 err = -EAGAIN; 2958 skb = sock_dequeue_err_skb(sk); 2959 if (skb == NULL) 2960 goto out; 2961 2962 copied = skb->len; 2963 if (copied > len) { 2964 msg->msg_flags |= MSG_TRUNC; 2965 copied = len; 2966 } 2967 err = skb_copy_datagram_msg(skb, 0, msg, copied); 2968 if (err) 2969 goto out_free_skb; 2970 2971 sock_recv_timestamp(msg, sk, skb); 2972 2973 serr = SKB_EXT_ERR(skb); 2974 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 2975 2976 msg->msg_flags |= MSG_ERRQUEUE; 2977 err = copied; 2978 2979 out_free_skb: 2980 kfree_skb(skb); 2981 out: 2982 return err; 2983 } 2984 EXPORT_SYMBOL(sock_recv_errqueue); 2985 2986 /* 2987 * Get a socket option on an socket. 2988 * 2989 * FIX: POSIX 1003.1g is very ambiguous here. It states that 2990 * asynchronous errors should be reported by getsockopt. We assume 2991 * this means if you specify SO_ERROR (otherwise whats the point of it). 2992 */ 2993 int sock_common_getsockopt(struct socket *sock, int level, int optname, 2994 char __user *optval, int __user *optlen) 2995 { 2996 struct sock *sk = sock->sk; 2997 2998 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 2999 } 3000 EXPORT_SYMBOL(sock_common_getsockopt); 3001 3002 #ifdef CONFIG_COMPAT 3003 int compat_sock_common_getsockopt(struct socket *sock, int level, int optname, 3004 char __user *optval, int __user *optlen) 3005 { 3006 struct sock *sk = sock->sk; 3007 3008 if (sk->sk_prot->compat_getsockopt != NULL) 3009 return sk->sk_prot->compat_getsockopt(sk, level, optname, 3010 optval, optlen); 3011 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 3012 } 3013 EXPORT_SYMBOL(compat_sock_common_getsockopt); 3014 #endif 3015 3016 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 3017 int flags) 3018 { 3019 struct sock *sk = sock->sk; 3020 int addr_len = 0; 3021 int err; 3022 3023 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, 3024 flags & ~MSG_DONTWAIT, &addr_len); 3025 if (err >= 0) 3026 msg->msg_namelen = addr_len; 3027 return err; 3028 } 3029 EXPORT_SYMBOL(sock_common_recvmsg); 3030 3031 /* 3032 * Set socket options on an inet socket. 3033 */ 3034 int sock_common_setsockopt(struct socket *sock, int level, int optname, 3035 char __user *optval, unsigned int optlen) 3036 { 3037 struct sock *sk = sock->sk; 3038 3039 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3040 } 3041 EXPORT_SYMBOL(sock_common_setsockopt); 3042 3043 #ifdef CONFIG_COMPAT 3044 int compat_sock_common_setsockopt(struct socket *sock, int level, int optname, 3045 char __user *optval, unsigned int optlen) 3046 { 3047 struct sock *sk = sock->sk; 3048 3049 if (sk->sk_prot->compat_setsockopt != NULL) 3050 return sk->sk_prot->compat_setsockopt(sk, level, optname, 3051 optval, optlen); 3052 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3053 } 3054 EXPORT_SYMBOL(compat_sock_common_setsockopt); 3055 #endif 3056 3057 void sk_common_release(struct sock *sk) 3058 { 3059 if (sk->sk_prot->destroy) 3060 sk->sk_prot->destroy(sk); 3061 3062 /* 3063 * Observation: when sock_common_release is called, processes have 3064 * no access to socket. But net still has. 3065 * Step one, detach it from networking: 3066 * 3067 * A. Remove from hash tables. 3068 */ 3069 3070 sk->sk_prot->unhash(sk); 3071 3072 /* 3073 * In this point socket cannot receive new packets, but it is possible 3074 * that some packets are in flight because some CPU runs receiver and 3075 * did hash table lookup before we unhashed socket. They will achieve 3076 * receive queue and will be purged by socket destructor. 3077 * 3078 * Also we still have packets pending on receive queue and probably, 3079 * our own packets waiting in device queues. sock_destroy will drain 3080 * receive queue, but transmitted packets will delay socket destruction 3081 * until the last reference will be released. 3082 */ 3083 3084 sock_orphan(sk); 3085 3086 xfrm_sk_free_policy(sk); 3087 3088 sk_refcnt_debug_release(sk); 3089 3090 sock_put(sk); 3091 } 3092 EXPORT_SYMBOL(sk_common_release); 3093 3094 void sk_get_meminfo(const struct sock *sk, u32 *mem) 3095 { 3096 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 3097 3098 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 3099 mem[SK_MEMINFO_RCVBUF] = sk->sk_rcvbuf; 3100 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 3101 mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf; 3102 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc; 3103 mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued; 3104 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 3105 mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len; 3106 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops); 3107 } 3108 3109 #ifdef CONFIG_PROC_FS 3110 #define PROTO_INUSE_NR 64 /* should be enough for the first time */ 3111 struct prot_inuse { 3112 int val[PROTO_INUSE_NR]; 3113 }; 3114 3115 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 3116 3117 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) 3118 { 3119 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); 3120 } 3121 EXPORT_SYMBOL_GPL(sock_prot_inuse_add); 3122 3123 int sock_prot_inuse_get(struct net *net, struct proto *prot) 3124 { 3125 int cpu, idx = prot->inuse_idx; 3126 int res = 0; 3127 3128 for_each_possible_cpu(cpu) 3129 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 3130 3131 return res >= 0 ? res : 0; 3132 } 3133 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 3134 3135 static void sock_inuse_add(struct net *net, int val) 3136 { 3137 this_cpu_add(*net->core.sock_inuse, val); 3138 } 3139 3140 int sock_inuse_get(struct net *net) 3141 { 3142 int cpu, res = 0; 3143 3144 for_each_possible_cpu(cpu) 3145 res += *per_cpu_ptr(net->core.sock_inuse, cpu); 3146 3147 return res; 3148 } 3149 3150 EXPORT_SYMBOL_GPL(sock_inuse_get); 3151 3152 static int __net_init sock_inuse_init_net(struct net *net) 3153 { 3154 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 3155 if (net->core.prot_inuse == NULL) 3156 return -ENOMEM; 3157 3158 net->core.sock_inuse = alloc_percpu(int); 3159 if (net->core.sock_inuse == NULL) 3160 goto out; 3161 3162 return 0; 3163 3164 out: 3165 free_percpu(net->core.prot_inuse); 3166 return -ENOMEM; 3167 } 3168 3169 static void __net_exit sock_inuse_exit_net(struct net *net) 3170 { 3171 free_percpu(net->core.prot_inuse); 3172 free_percpu(net->core.sock_inuse); 3173 } 3174 3175 static struct pernet_operations net_inuse_ops = { 3176 .init = sock_inuse_init_net, 3177 .exit = sock_inuse_exit_net, 3178 }; 3179 3180 static __init int net_inuse_init(void) 3181 { 3182 if (register_pernet_subsys(&net_inuse_ops)) 3183 panic("Cannot initialize net inuse counters"); 3184 3185 return 0; 3186 } 3187 3188 core_initcall(net_inuse_init); 3189 3190 static void assign_proto_idx(struct proto *prot) 3191 { 3192 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 3193 3194 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 3195 pr_err("PROTO_INUSE_NR exhausted\n"); 3196 return; 3197 } 3198 3199 set_bit(prot->inuse_idx, proto_inuse_idx); 3200 } 3201 3202 static void release_proto_idx(struct proto *prot) 3203 { 3204 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 3205 clear_bit(prot->inuse_idx, proto_inuse_idx); 3206 } 3207 #else 3208 static inline void assign_proto_idx(struct proto *prot) 3209 { 3210 } 3211 3212 static inline void release_proto_idx(struct proto *prot) 3213 { 3214 } 3215 3216 static void sock_inuse_add(struct net *net, int val) 3217 { 3218 } 3219 #endif 3220 3221 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 3222 { 3223 if (!rsk_prot) 3224 return; 3225 kfree(rsk_prot->slab_name); 3226 rsk_prot->slab_name = NULL; 3227 kmem_cache_destroy(rsk_prot->slab); 3228 rsk_prot->slab = NULL; 3229 } 3230 3231 static int req_prot_init(const struct proto *prot) 3232 { 3233 struct request_sock_ops *rsk_prot = prot->rsk_prot; 3234 3235 if (!rsk_prot) 3236 return 0; 3237 3238 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 3239 prot->name); 3240 if (!rsk_prot->slab_name) 3241 return -ENOMEM; 3242 3243 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 3244 rsk_prot->obj_size, 0, 3245 SLAB_ACCOUNT | prot->slab_flags, 3246 NULL); 3247 3248 if (!rsk_prot->slab) { 3249 pr_crit("%s: Can't create request sock SLAB cache!\n", 3250 prot->name); 3251 return -ENOMEM; 3252 } 3253 return 0; 3254 } 3255 3256 int proto_register(struct proto *prot, int alloc_slab) 3257 { 3258 if (alloc_slab) { 3259 prot->slab = kmem_cache_create_usercopy(prot->name, 3260 prot->obj_size, 0, 3261 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 3262 prot->slab_flags, 3263 prot->useroffset, prot->usersize, 3264 NULL); 3265 3266 if (prot->slab == NULL) { 3267 pr_crit("%s: Can't create sock SLAB cache!\n", 3268 prot->name); 3269 goto out; 3270 } 3271 3272 if (req_prot_init(prot)) 3273 goto out_free_request_sock_slab; 3274 3275 if (prot->twsk_prot != NULL) { 3276 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name); 3277 3278 if (prot->twsk_prot->twsk_slab_name == NULL) 3279 goto out_free_request_sock_slab; 3280 3281 prot->twsk_prot->twsk_slab = 3282 kmem_cache_create(prot->twsk_prot->twsk_slab_name, 3283 prot->twsk_prot->twsk_obj_size, 3284 0, 3285 SLAB_ACCOUNT | 3286 prot->slab_flags, 3287 NULL); 3288 if (prot->twsk_prot->twsk_slab == NULL) 3289 goto out_free_timewait_sock_slab_name; 3290 } 3291 } 3292 3293 mutex_lock(&proto_list_mutex); 3294 list_add(&prot->node, &proto_list); 3295 assign_proto_idx(prot); 3296 mutex_unlock(&proto_list_mutex); 3297 return 0; 3298 3299 out_free_timewait_sock_slab_name: 3300 kfree(prot->twsk_prot->twsk_slab_name); 3301 out_free_request_sock_slab: 3302 req_prot_cleanup(prot->rsk_prot); 3303 3304 kmem_cache_destroy(prot->slab); 3305 prot->slab = NULL; 3306 out: 3307 return -ENOBUFS; 3308 } 3309 EXPORT_SYMBOL(proto_register); 3310 3311 void proto_unregister(struct proto *prot) 3312 { 3313 mutex_lock(&proto_list_mutex); 3314 release_proto_idx(prot); 3315 list_del(&prot->node); 3316 mutex_unlock(&proto_list_mutex); 3317 3318 kmem_cache_destroy(prot->slab); 3319 prot->slab = NULL; 3320 3321 req_prot_cleanup(prot->rsk_prot); 3322 3323 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) { 3324 kmem_cache_destroy(prot->twsk_prot->twsk_slab); 3325 kfree(prot->twsk_prot->twsk_slab_name); 3326 prot->twsk_prot->twsk_slab = NULL; 3327 } 3328 } 3329 EXPORT_SYMBOL(proto_unregister); 3330 3331 int sock_load_diag_module(int family, int protocol) 3332 { 3333 if (!protocol) { 3334 if (!sock_is_registered(family)) 3335 return -ENOENT; 3336 3337 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 3338 NETLINK_SOCK_DIAG, family); 3339 } 3340 3341 #ifdef CONFIG_INET 3342 if (family == AF_INET && 3343 !rcu_access_pointer(inet_protos[protocol])) 3344 return -ENOENT; 3345 #endif 3346 3347 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 3348 NETLINK_SOCK_DIAG, family, protocol); 3349 } 3350 EXPORT_SYMBOL(sock_load_diag_module); 3351 3352 #ifdef CONFIG_PROC_FS 3353 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 3354 __acquires(proto_list_mutex) 3355 { 3356 mutex_lock(&proto_list_mutex); 3357 return seq_list_start_head(&proto_list, *pos); 3358 } 3359 3360 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3361 { 3362 return seq_list_next(v, &proto_list, pos); 3363 } 3364 3365 static void proto_seq_stop(struct seq_file *seq, void *v) 3366 __releases(proto_list_mutex) 3367 { 3368 mutex_unlock(&proto_list_mutex); 3369 } 3370 3371 static char proto_method_implemented(const void *method) 3372 { 3373 return method == NULL ? 'n' : 'y'; 3374 } 3375 static long sock_prot_memory_allocated(struct proto *proto) 3376 { 3377 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 3378 } 3379 3380 static char *sock_prot_memory_pressure(struct proto *proto) 3381 { 3382 return proto->memory_pressure != NULL ? 3383 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 3384 } 3385 3386 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 3387 { 3388 3389 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 3390 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 3391 proto->name, 3392 proto->obj_size, 3393 sock_prot_inuse_get(seq_file_net(seq), proto), 3394 sock_prot_memory_allocated(proto), 3395 sock_prot_memory_pressure(proto), 3396 proto->max_header, 3397 proto->slab == NULL ? "no" : "yes", 3398 module_name(proto->owner), 3399 proto_method_implemented(proto->close), 3400 proto_method_implemented(proto->connect), 3401 proto_method_implemented(proto->disconnect), 3402 proto_method_implemented(proto->accept), 3403 proto_method_implemented(proto->ioctl), 3404 proto_method_implemented(proto->init), 3405 proto_method_implemented(proto->destroy), 3406 proto_method_implemented(proto->shutdown), 3407 proto_method_implemented(proto->setsockopt), 3408 proto_method_implemented(proto->getsockopt), 3409 proto_method_implemented(proto->sendmsg), 3410 proto_method_implemented(proto->recvmsg), 3411 proto_method_implemented(proto->sendpage), 3412 proto_method_implemented(proto->bind), 3413 proto_method_implemented(proto->backlog_rcv), 3414 proto_method_implemented(proto->hash), 3415 proto_method_implemented(proto->unhash), 3416 proto_method_implemented(proto->get_port), 3417 proto_method_implemented(proto->enter_memory_pressure)); 3418 } 3419 3420 static int proto_seq_show(struct seq_file *seq, void *v) 3421 { 3422 if (v == &proto_list) 3423 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 3424 "protocol", 3425 "size", 3426 "sockets", 3427 "memory", 3428 "press", 3429 "maxhdr", 3430 "slab", 3431 "module", 3432 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n"); 3433 else 3434 proto_seq_printf(seq, list_entry(v, struct proto, node)); 3435 return 0; 3436 } 3437 3438 static const struct seq_operations proto_seq_ops = { 3439 .start = proto_seq_start, 3440 .next = proto_seq_next, 3441 .stop = proto_seq_stop, 3442 .show = proto_seq_show, 3443 }; 3444 3445 static __net_init int proto_init_net(struct net *net) 3446 { 3447 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 3448 sizeof(struct seq_net_private))) 3449 return -ENOMEM; 3450 3451 return 0; 3452 } 3453 3454 static __net_exit void proto_exit_net(struct net *net) 3455 { 3456 remove_proc_entry("protocols", net->proc_net); 3457 } 3458 3459 3460 static __net_initdata struct pernet_operations proto_net_ops = { 3461 .init = proto_init_net, 3462 .exit = proto_exit_net, 3463 }; 3464 3465 static int __init proto_init(void) 3466 { 3467 return register_pernet_subsys(&proto_net_ops); 3468 } 3469 3470 subsys_initcall(proto_init); 3471 3472 #endif /* PROC_FS */ 3473 3474 #ifdef CONFIG_NET_RX_BUSY_POLL 3475 bool sk_busy_loop_end(void *p, unsigned long start_time) 3476 { 3477 struct sock *sk = p; 3478 3479 return !skb_queue_empty(&sk->sk_receive_queue) || 3480 sk_busy_loop_timeout(sk, start_time); 3481 } 3482 EXPORT_SYMBOL(sk_busy_loop_end); 3483 #endif /* CONFIG_NET_RX_BUSY_POLL */ 3484