1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Generic socket support routines. Memory allocators, socket lock/release 7 * handler for protocols to use and generic option handler. 8 * 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 * 85 * 86 * This program is free software; you can redistribute it and/or 87 * modify it under the terms of the GNU General Public License 88 * as published by the Free Software Foundation; either version 89 * 2 of the License, or (at your option) any later version. 90 */ 91 92 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 93 94 #include <linux/capability.h> 95 #include <linux/errno.h> 96 #include <linux/errqueue.h> 97 #include <linux/types.h> 98 #include <linux/socket.h> 99 #include <linux/in.h> 100 #include <linux/kernel.h> 101 #include <linux/module.h> 102 #include <linux/proc_fs.h> 103 #include <linux/seq_file.h> 104 #include <linux/sched.h> 105 #include <linux/timer.h> 106 #include <linux/string.h> 107 #include <linux/sockios.h> 108 #include <linux/net.h> 109 #include <linux/mm.h> 110 #include <linux/slab.h> 111 #include <linux/interrupt.h> 112 #include <linux/poll.h> 113 #include <linux/tcp.h> 114 #include <linux/init.h> 115 #include <linux/highmem.h> 116 #include <linux/user_namespace.h> 117 #include <linux/static_key.h> 118 #include <linux/memcontrol.h> 119 #include <linux/prefetch.h> 120 121 #include <linux/uaccess.h> 122 123 #include <linux/netdevice.h> 124 #include <net/protocol.h> 125 #include <linux/skbuff.h> 126 #include <net/net_namespace.h> 127 #include <net/request_sock.h> 128 #include <net/sock.h> 129 #include <linux/net_tstamp.h> 130 #include <net/xfrm.h> 131 #include <linux/ipsec.h> 132 #include <net/cls_cgroup.h> 133 #include <net/netprio_cgroup.h> 134 #include <linux/sock_diag.h> 135 136 #include <linux/filter.h> 137 #include <net/sock_reuseport.h> 138 139 #include <trace/events/sock.h> 140 141 #ifdef CONFIG_INET 142 #include <net/tcp.h> 143 #endif 144 145 #include <net/busy_poll.h> 146 147 static DEFINE_MUTEX(proto_list_mutex); 148 static LIST_HEAD(proto_list); 149 150 /** 151 * sk_ns_capable - General socket capability test 152 * @sk: Socket to use a capability on or through 153 * @user_ns: The user namespace of the capability to use 154 * @cap: The capability to use 155 * 156 * Test to see if the opener of the socket had when the socket was 157 * created and the current process has the capability @cap in the user 158 * namespace @user_ns. 159 */ 160 bool sk_ns_capable(const struct sock *sk, 161 struct user_namespace *user_ns, int cap) 162 { 163 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 164 ns_capable(user_ns, cap); 165 } 166 EXPORT_SYMBOL(sk_ns_capable); 167 168 /** 169 * sk_capable - Socket global capability test 170 * @sk: Socket to use a capability on or through 171 * @cap: The global capability to use 172 * 173 * Test to see if the opener of the socket had when the socket was 174 * created and the current process has the capability @cap in all user 175 * namespaces. 176 */ 177 bool sk_capable(const struct sock *sk, int cap) 178 { 179 return sk_ns_capable(sk, &init_user_ns, cap); 180 } 181 EXPORT_SYMBOL(sk_capable); 182 183 /** 184 * sk_net_capable - Network namespace socket capability test 185 * @sk: Socket to use a capability on or through 186 * @cap: The capability to use 187 * 188 * Test to see if the opener of the socket had when the socket was created 189 * and the current process has the capability @cap over the network namespace 190 * the socket is a member of. 191 */ 192 bool sk_net_capable(const struct sock *sk, int cap) 193 { 194 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 195 } 196 EXPORT_SYMBOL(sk_net_capable); 197 198 /* 199 * Each address family might have different locking rules, so we have 200 * one slock key per address family: 201 */ 202 static struct lock_class_key af_family_keys[AF_MAX]; 203 static struct lock_class_key af_family_slock_keys[AF_MAX]; 204 205 /* 206 * Make lock validator output more readable. (we pre-construct these 207 * strings build-time, so that runtime initialization of socket 208 * locks is fast): 209 */ 210 static const char *const af_family_key_strings[AF_MAX+1] = { 211 "sk_lock-AF_UNSPEC", "sk_lock-AF_UNIX" , "sk_lock-AF_INET" , 212 "sk_lock-AF_AX25" , "sk_lock-AF_IPX" , "sk_lock-AF_APPLETALK", 213 "sk_lock-AF_NETROM", "sk_lock-AF_BRIDGE" , "sk_lock-AF_ATMPVC" , 214 "sk_lock-AF_X25" , "sk_lock-AF_INET6" , "sk_lock-AF_ROSE" , 215 "sk_lock-AF_DECnet", "sk_lock-AF_NETBEUI" , "sk_lock-AF_SECURITY" , 216 "sk_lock-AF_KEY" , "sk_lock-AF_NETLINK" , "sk_lock-AF_PACKET" , 217 "sk_lock-AF_ASH" , "sk_lock-AF_ECONET" , "sk_lock-AF_ATMSVC" , 218 "sk_lock-AF_RDS" , "sk_lock-AF_SNA" , "sk_lock-AF_IRDA" , 219 "sk_lock-AF_PPPOX" , "sk_lock-AF_WANPIPE" , "sk_lock-AF_LLC" , 220 "sk_lock-27" , "sk_lock-28" , "sk_lock-AF_CAN" , 221 "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-IUCV" , 222 "sk_lock-AF_RXRPC" , "sk_lock-AF_ISDN" , "sk_lock-AF_PHONET" , 223 "sk_lock-AF_IEEE802154", "sk_lock-AF_CAIF" , "sk_lock-AF_ALG" , 224 "sk_lock-AF_NFC" , "sk_lock-AF_VSOCK" , "sk_lock-AF_KCM" , 225 "sk_lock-AF_QIPCRTR", "sk_lock-AF_SMC" , "sk_lock-AF_MAX" 226 }; 227 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 228 "slock-AF_UNSPEC", "slock-AF_UNIX" , "slock-AF_INET" , 229 "slock-AF_AX25" , "slock-AF_IPX" , "slock-AF_APPLETALK", 230 "slock-AF_NETROM", "slock-AF_BRIDGE" , "slock-AF_ATMPVC" , 231 "slock-AF_X25" , "slock-AF_INET6" , "slock-AF_ROSE" , 232 "slock-AF_DECnet", "slock-AF_NETBEUI" , "slock-AF_SECURITY" , 233 "slock-AF_KEY" , "slock-AF_NETLINK" , "slock-AF_PACKET" , 234 "slock-AF_ASH" , "slock-AF_ECONET" , "slock-AF_ATMSVC" , 235 "slock-AF_RDS" , "slock-AF_SNA" , "slock-AF_IRDA" , 236 "slock-AF_PPPOX" , "slock-AF_WANPIPE" , "slock-AF_LLC" , 237 "slock-27" , "slock-28" , "slock-AF_CAN" , 238 "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_IUCV" , 239 "slock-AF_RXRPC" , "slock-AF_ISDN" , "slock-AF_PHONET" , 240 "slock-AF_IEEE802154", "slock-AF_CAIF" , "slock-AF_ALG" , 241 "slock-AF_NFC" , "slock-AF_VSOCK" ,"slock-AF_KCM" , 242 "slock-AF_QIPCRTR", "slock-AF_SMC" , "slock-AF_MAX" 243 }; 244 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 245 "clock-AF_UNSPEC", "clock-AF_UNIX" , "clock-AF_INET" , 246 "clock-AF_AX25" , "clock-AF_IPX" , "clock-AF_APPLETALK", 247 "clock-AF_NETROM", "clock-AF_BRIDGE" , "clock-AF_ATMPVC" , 248 "clock-AF_X25" , "clock-AF_INET6" , "clock-AF_ROSE" , 249 "clock-AF_DECnet", "clock-AF_NETBEUI" , "clock-AF_SECURITY" , 250 "clock-AF_KEY" , "clock-AF_NETLINK" , "clock-AF_PACKET" , 251 "clock-AF_ASH" , "clock-AF_ECONET" , "clock-AF_ATMSVC" , 252 "clock-AF_RDS" , "clock-AF_SNA" , "clock-AF_IRDA" , 253 "clock-AF_PPPOX" , "clock-AF_WANPIPE" , "clock-AF_LLC" , 254 "clock-27" , "clock-28" , "clock-AF_CAN" , 255 "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" , 256 "clock-AF_RXRPC" , "clock-AF_ISDN" , "clock-AF_PHONET" , 257 "clock-AF_IEEE802154", "clock-AF_CAIF" , "clock-AF_ALG" , 258 "clock-AF_NFC" , "clock-AF_VSOCK" , "clock-AF_KCM" , 259 "clock-AF_QIPCRTR", "clock-AF_SMC" , "clock-AF_MAX" 260 }; 261 262 /* 263 * sk_callback_lock locking rules are per-address-family, 264 * so split the lock classes by using a per-AF key: 265 */ 266 static struct lock_class_key af_callback_keys[AF_MAX]; 267 268 /* Take into consideration the size of the struct sk_buff overhead in the 269 * determination of these values, since that is non-constant across 270 * platforms. This makes socket queueing behavior and performance 271 * not depend upon such differences. 272 */ 273 #define _SK_MEM_PACKETS 256 274 #define _SK_MEM_OVERHEAD SKB_TRUESIZE(256) 275 #define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS) 276 #define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS) 277 278 /* Run time adjustable parameters. */ 279 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 280 EXPORT_SYMBOL(sysctl_wmem_max); 281 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 282 EXPORT_SYMBOL(sysctl_rmem_max); 283 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 284 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 285 286 /* Maximal space eaten by iovec or ancillary data plus some space */ 287 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 288 EXPORT_SYMBOL(sysctl_optmem_max); 289 290 int sysctl_tstamp_allow_data __read_mostly = 1; 291 292 struct static_key memalloc_socks = STATIC_KEY_INIT_FALSE; 293 EXPORT_SYMBOL_GPL(memalloc_socks); 294 295 /** 296 * sk_set_memalloc - sets %SOCK_MEMALLOC 297 * @sk: socket to set it on 298 * 299 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 300 * It's the responsibility of the admin to adjust min_free_kbytes 301 * to meet the requirements 302 */ 303 void sk_set_memalloc(struct sock *sk) 304 { 305 sock_set_flag(sk, SOCK_MEMALLOC); 306 sk->sk_allocation |= __GFP_MEMALLOC; 307 static_key_slow_inc(&memalloc_socks); 308 } 309 EXPORT_SYMBOL_GPL(sk_set_memalloc); 310 311 void sk_clear_memalloc(struct sock *sk) 312 { 313 sock_reset_flag(sk, SOCK_MEMALLOC); 314 sk->sk_allocation &= ~__GFP_MEMALLOC; 315 static_key_slow_dec(&memalloc_socks); 316 317 /* 318 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 319 * progress of swapping. SOCK_MEMALLOC may be cleared while 320 * it has rmem allocations due to the last swapfile being deactivated 321 * but there is a risk that the socket is unusable due to exceeding 322 * the rmem limits. Reclaim the reserves and obey rmem limits again. 323 */ 324 sk_mem_reclaim(sk); 325 } 326 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 327 328 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 329 { 330 int ret; 331 unsigned long pflags = current->flags; 332 333 /* these should have been dropped before queueing */ 334 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 335 336 current->flags |= PF_MEMALLOC; 337 ret = sk->sk_backlog_rcv(sk, skb); 338 tsk_restore_flags(current, pflags, PF_MEMALLOC); 339 340 return ret; 341 } 342 EXPORT_SYMBOL(__sk_backlog_rcv); 343 344 static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen) 345 { 346 struct timeval tv; 347 348 if (optlen < sizeof(tv)) 349 return -EINVAL; 350 if (copy_from_user(&tv, optval, sizeof(tv))) 351 return -EFAULT; 352 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 353 return -EDOM; 354 355 if (tv.tv_sec < 0) { 356 static int warned __read_mostly; 357 358 *timeo_p = 0; 359 if (warned < 10 && net_ratelimit()) { 360 warned++; 361 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 362 __func__, current->comm, task_pid_nr(current)); 363 } 364 return 0; 365 } 366 *timeo_p = MAX_SCHEDULE_TIMEOUT; 367 if (tv.tv_sec == 0 && tv.tv_usec == 0) 368 return 0; 369 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1)) 370 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ); 371 return 0; 372 } 373 374 static void sock_warn_obsolete_bsdism(const char *name) 375 { 376 static int warned; 377 static char warncomm[TASK_COMM_LEN]; 378 if (strcmp(warncomm, current->comm) && warned < 5) { 379 strcpy(warncomm, current->comm); 380 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n", 381 warncomm, name); 382 warned++; 383 } 384 } 385 386 static bool sock_needs_netstamp(const struct sock *sk) 387 { 388 switch (sk->sk_family) { 389 case AF_UNSPEC: 390 case AF_UNIX: 391 return false; 392 default: 393 return true; 394 } 395 } 396 397 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 398 { 399 if (sk->sk_flags & flags) { 400 sk->sk_flags &= ~flags; 401 if (sock_needs_netstamp(sk) && 402 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 403 net_disable_timestamp(); 404 } 405 } 406 407 408 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 409 { 410 unsigned long flags; 411 struct sk_buff_head *list = &sk->sk_receive_queue; 412 413 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { 414 atomic_inc(&sk->sk_drops); 415 trace_sock_rcvqueue_full(sk, skb); 416 return -ENOMEM; 417 } 418 419 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 420 atomic_inc(&sk->sk_drops); 421 return -ENOBUFS; 422 } 423 424 skb->dev = NULL; 425 skb_set_owner_r(skb, sk); 426 427 /* we escape from rcu protected region, make sure we dont leak 428 * a norefcounted dst 429 */ 430 skb_dst_force(skb); 431 432 spin_lock_irqsave(&list->lock, flags); 433 sock_skb_set_dropcount(sk, skb); 434 __skb_queue_tail(list, skb); 435 spin_unlock_irqrestore(&list->lock, flags); 436 437 if (!sock_flag(sk, SOCK_DEAD)) 438 sk->sk_data_ready(sk); 439 return 0; 440 } 441 EXPORT_SYMBOL(__sock_queue_rcv_skb); 442 443 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 444 { 445 int err; 446 447 err = sk_filter(sk, skb); 448 if (err) 449 return err; 450 451 return __sock_queue_rcv_skb(sk, skb); 452 } 453 EXPORT_SYMBOL(sock_queue_rcv_skb); 454 455 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 456 const int nested, unsigned int trim_cap, bool refcounted) 457 { 458 int rc = NET_RX_SUCCESS; 459 460 if (sk_filter_trim_cap(sk, skb, trim_cap)) 461 goto discard_and_relse; 462 463 skb->dev = NULL; 464 465 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) { 466 atomic_inc(&sk->sk_drops); 467 goto discard_and_relse; 468 } 469 if (nested) 470 bh_lock_sock_nested(sk); 471 else 472 bh_lock_sock(sk); 473 if (!sock_owned_by_user(sk)) { 474 /* 475 * trylock + unlock semantics: 476 */ 477 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 478 479 rc = sk_backlog_rcv(sk, skb); 480 481 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_); 482 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) { 483 bh_unlock_sock(sk); 484 atomic_inc(&sk->sk_drops); 485 goto discard_and_relse; 486 } 487 488 bh_unlock_sock(sk); 489 out: 490 if (refcounted) 491 sock_put(sk); 492 return rc; 493 discard_and_relse: 494 kfree_skb(skb); 495 goto out; 496 } 497 EXPORT_SYMBOL(__sk_receive_skb); 498 499 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 500 { 501 struct dst_entry *dst = __sk_dst_get(sk); 502 503 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 504 sk_tx_queue_clear(sk); 505 sk->sk_dst_pending_confirm = 0; 506 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 507 dst_release(dst); 508 return NULL; 509 } 510 511 return dst; 512 } 513 EXPORT_SYMBOL(__sk_dst_check); 514 515 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 516 { 517 struct dst_entry *dst = sk_dst_get(sk); 518 519 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 520 sk_dst_reset(sk); 521 dst_release(dst); 522 return NULL; 523 } 524 525 return dst; 526 } 527 EXPORT_SYMBOL(sk_dst_check); 528 529 static int sock_setbindtodevice(struct sock *sk, char __user *optval, 530 int optlen) 531 { 532 int ret = -ENOPROTOOPT; 533 #ifdef CONFIG_NETDEVICES 534 struct net *net = sock_net(sk); 535 char devname[IFNAMSIZ]; 536 int index; 537 538 /* Sorry... */ 539 ret = -EPERM; 540 if (!ns_capable(net->user_ns, CAP_NET_RAW)) 541 goto out; 542 543 ret = -EINVAL; 544 if (optlen < 0) 545 goto out; 546 547 /* Bind this socket to a particular device like "eth0", 548 * as specified in the passed interface name. If the 549 * name is "" or the option length is zero the socket 550 * is not bound. 551 */ 552 if (optlen > IFNAMSIZ - 1) 553 optlen = IFNAMSIZ - 1; 554 memset(devname, 0, sizeof(devname)); 555 556 ret = -EFAULT; 557 if (copy_from_user(devname, optval, optlen)) 558 goto out; 559 560 index = 0; 561 if (devname[0] != '\0') { 562 struct net_device *dev; 563 564 rcu_read_lock(); 565 dev = dev_get_by_name_rcu(net, devname); 566 if (dev) 567 index = dev->ifindex; 568 rcu_read_unlock(); 569 ret = -ENODEV; 570 if (!dev) 571 goto out; 572 } 573 574 lock_sock(sk); 575 sk->sk_bound_dev_if = index; 576 sk_dst_reset(sk); 577 release_sock(sk); 578 579 ret = 0; 580 581 out: 582 #endif 583 584 return ret; 585 } 586 587 static int sock_getbindtodevice(struct sock *sk, char __user *optval, 588 int __user *optlen, int len) 589 { 590 int ret = -ENOPROTOOPT; 591 #ifdef CONFIG_NETDEVICES 592 struct net *net = sock_net(sk); 593 char devname[IFNAMSIZ]; 594 595 if (sk->sk_bound_dev_if == 0) { 596 len = 0; 597 goto zero; 598 } 599 600 ret = -EINVAL; 601 if (len < IFNAMSIZ) 602 goto out; 603 604 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if); 605 if (ret) 606 goto out; 607 608 len = strlen(devname) + 1; 609 610 ret = -EFAULT; 611 if (copy_to_user(optval, devname, len)) 612 goto out; 613 614 zero: 615 ret = -EFAULT; 616 if (put_user(len, optlen)) 617 goto out; 618 619 ret = 0; 620 621 out: 622 #endif 623 624 return ret; 625 } 626 627 static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool) 628 { 629 if (valbool) 630 sock_set_flag(sk, bit); 631 else 632 sock_reset_flag(sk, bit); 633 } 634 635 bool sk_mc_loop(struct sock *sk) 636 { 637 if (dev_recursion_level()) 638 return false; 639 if (!sk) 640 return true; 641 switch (sk->sk_family) { 642 case AF_INET: 643 return inet_sk(sk)->mc_loop; 644 #if IS_ENABLED(CONFIG_IPV6) 645 case AF_INET6: 646 return inet6_sk(sk)->mc_loop; 647 #endif 648 } 649 WARN_ON(1); 650 return true; 651 } 652 EXPORT_SYMBOL(sk_mc_loop); 653 654 /* 655 * This is meant for all protocols to use and covers goings on 656 * at the socket level. Everything here is generic. 657 */ 658 659 int sock_setsockopt(struct socket *sock, int level, int optname, 660 char __user *optval, unsigned int optlen) 661 { 662 struct sock *sk = sock->sk; 663 int val; 664 int valbool; 665 struct linger ling; 666 int ret = 0; 667 668 /* 669 * Options without arguments 670 */ 671 672 if (optname == SO_BINDTODEVICE) 673 return sock_setbindtodevice(sk, optval, optlen); 674 675 if (optlen < sizeof(int)) 676 return -EINVAL; 677 678 if (get_user(val, (int __user *)optval)) 679 return -EFAULT; 680 681 valbool = val ? 1 : 0; 682 683 lock_sock(sk); 684 685 switch (optname) { 686 case SO_DEBUG: 687 if (val && !capable(CAP_NET_ADMIN)) 688 ret = -EACCES; 689 else 690 sock_valbool_flag(sk, SOCK_DBG, valbool); 691 break; 692 case SO_REUSEADDR: 693 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 694 break; 695 case SO_REUSEPORT: 696 sk->sk_reuseport = valbool; 697 break; 698 case SO_TYPE: 699 case SO_PROTOCOL: 700 case SO_DOMAIN: 701 case SO_ERROR: 702 ret = -ENOPROTOOPT; 703 break; 704 case SO_DONTROUTE: 705 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 706 break; 707 case SO_BROADCAST: 708 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 709 break; 710 case SO_SNDBUF: 711 /* Don't error on this BSD doesn't and if you think 712 * about it this is right. Otherwise apps have to 713 * play 'guess the biggest size' games. RCVBUF/SNDBUF 714 * are treated in BSD as hints 715 */ 716 val = min_t(u32, val, sysctl_wmem_max); 717 set_sndbuf: 718 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 719 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); 720 /* Wake up sending tasks if we upped the value. */ 721 sk->sk_write_space(sk); 722 break; 723 724 case SO_SNDBUFFORCE: 725 if (!capable(CAP_NET_ADMIN)) { 726 ret = -EPERM; 727 break; 728 } 729 goto set_sndbuf; 730 731 case SO_RCVBUF: 732 /* Don't error on this BSD doesn't and if you think 733 * about it this is right. Otherwise apps have to 734 * play 'guess the biggest size' games. RCVBUF/SNDBUF 735 * are treated in BSD as hints 736 */ 737 val = min_t(u32, val, sysctl_rmem_max); 738 set_rcvbuf: 739 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 740 /* 741 * We double it on the way in to account for 742 * "struct sk_buff" etc. overhead. Applications 743 * assume that the SO_RCVBUF setting they make will 744 * allow that much actual data to be received on that 745 * socket. 746 * 747 * Applications are unaware that "struct sk_buff" and 748 * other overheads allocate from the receive buffer 749 * during socket buffer allocation. 750 * 751 * And after considering the possible alternatives, 752 * returning the value we actually used in getsockopt 753 * is the most desirable behavior. 754 */ 755 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF); 756 break; 757 758 case SO_RCVBUFFORCE: 759 if (!capable(CAP_NET_ADMIN)) { 760 ret = -EPERM; 761 break; 762 } 763 goto set_rcvbuf; 764 765 case SO_KEEPALIVE: 766 if (sk->sk_prot->keepalive) 767 sk->sk_prot->keepalive(sk, valbool); 768 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 769 break; 770 771 case SO_OOBINLINE: 772 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 773 break; 774 775 case SO_NO_CHECK: 776 sk->sk_no_check_tx = valbool; 777 break; 778 779 case SO_PRIORITY: 780 if ((val >= 0 && val <= 6) || 781 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 782 sk->sk_priority = val; 783 else 784 ret = -EPERM; 785 break; 786 787 case SO_LINGER: 788 if (optlen < sizeof(ling)) { 789 ret = -EINVAL; /* 1003.1g */ 790 break; 791 } 792 if (copy_from_user(&ling, optval, sizeof(ling))) { 793 ret = -EFAULT; 794 break; 795 } 796 if (!ling.l_onoff) 797 sock_reset_flag(sk, SOCK_LINGER); 798 else { 799 #if (BITS_PER_LONG == 32) 800 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ) 801 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT; 802 else 803 #endif 804 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ; 805 sock_set_flag(sk, SOCK_LINGER); 806 } 807 break; 808 809 case SO_BSDCOMPAT: 810 sock_warn_obsolete_bsdism("setsockopt"); 811 break; 812 813 case SO_PASSCRED: 814 if (valbool) 815 set_bit(SOCK_PASSCRED, &sock->flags); 816 else 817 clear_bit(SOCK_PASSCRED, &sock->flags); 818 break; 819 820 case SO_TIMESTAMP: 821 case SO_TIMESTAMPNS: 822 if (valbool) { 823 if (optname == SO_TIMESTAMP) 824 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 825 else 826 sock_set_flag(sk, SOCK_RCVTSTAMPNS); 827 sock_set_flag(sk, SOCK_RCVTSTAMP); 828 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 829 } else { 830 sock_reset_flag(sk, SOCK_RCVTSTAMP); 831 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 832 } 833 break; 834 835 case SO_TIMESTAMPING: 836 if (val & ~SOF_TIMESTAMPING_MASK) { 837 ret = -EINVAL; 838 break; 839 } 840 841 if (val & SOF_TIMESTAMPING_OPT_ID && 842 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 843 if (sk->sk_protocol == IPPROTO_TCP && 844 sk->sk_type == SOCK_STREAM) { 845 if ((1 << sk->sk_state) & 846 (TCPF_CLOSE | TCPF_LISTEN)) { 847 ret = -EINVAL; 848 break; 849 } 850 sk->sk_tskey = tcp_sk(sk)->snd_una; 851 } else { 852 sk->sk_tskey = 0; 853 } 854 } 855 856 if (val & SOF_TIMESTAMPING_OPT_STATS && 857 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) { 858 ret = -EINVAL; 859 break; 860 } 861 862 sk->sk_tsflags = val; 863 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 864 sock_enable_timestamp(sk, 865 SOCK_TIMESTAMPING_RX_SOFTWARE); 866 else 867 sock_disable_timestamp(sk, 868 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 869 break; 870 871 case SO_RCVLOWAT: 872 if (val < 0) 873 val = INT_MAX; 874 sk->sk_rcvlowat = val ? : 1; 875 break; 876 877 case SO_RCVTIMEO: 878 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen); 879 break; 880 881 case SO_SNDTIMEO: 882 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen); 883 break; 884 885 case SO_ATTACH_FILTER: 886 ret = -EINVAL; 887 if (optlen == sizeof(struct sock_fprog)) { 888 struct sock_fprog fprog; 889 890 ret = -EFAULT; 891 if (copy_from_user(&fprog, optval, sizeof(fprog))) 892 break; 893 894 ret = sk_attach_filter(&fprog, sk); 895 } 896 break; 897 898 case SO_ATTACH_BPF: 899 ret = -EINVAL; 900 if (optlen == sizeof(u32)) { 901 u32 ufd; 902 903 ret = -EFAULT; 904 if (copy_from_user(&ufd, optval, sizeof(ufd))) 905 break; 906 907 ret = sk_attach_bpf(ufd, sk); 908 } 909 break; 910 911 case SO_ATTACH_REUSEPORT_CBPF: 912 ret = -EINVAL; 913 if (optlen == sizeof(struct sock_fprog)) { 914 struct sock_fprog fprog; 915 916 ret = -EFAULT; 917 if (copy_from_user(&fprog, optval, sizeof(fprog))) 918 break; 919 920 ret = sk_reuseport_attach_filter(&fprog, sk); 921 } 922 break; 923 924 case SO_ATTACH_REUSEPORT_EBPF: 925 ret = -EINVAL; 926 if (optlen == sizeof(u32)) { 927 u32 ufd; 928 929 ret = -EFAULT; 930 if (copy_from_user(&ufd, optval, sizeof(ufd))) 931 break; 932 933 ret = sk_reuseport_attach_bpf(ufd, sk); 934 } 935 break; 936 937 case SO_DETACH_FILTER: 938 ret = sk_detach_filter(sk); 939 break; 940 941 case SO_LOCK_FILTER: 942 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 943 ret = -EPERM; 944 else 945 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 946 break; 947 948 case SO_PASSSEC: 949 if (valbool) 950 set_bit(SOCK_PASSSEC, &sock->flags); 951 else 952 clear_bit(SOCK_PASSSEC, &sock->flags); 953 break; 954 case SO_MARK: 955 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 956 ret = -EPERM; 957 else 958 sk->sk_mark = val; 959 break; 960 961 case SO_RXQ_OVFL: 962 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 963 break; 964 965 case SO_WIFI_STATUS: 966 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 967 break; 968 969 case SO_PEEK_OFF: 970 if (sock->ops->set_peek_off) 971 ret = sock->ops->set_peek_off(sk, val); 972 else 973 ret = -EOPNOTSUPP; 974 break; 975 976 case SO_NOFCS: 977 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 978 break; 979 980 case SO_SELECT_ERR_QUEUE: 981 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 982 break; 983 984 #ifdef CONFIG_NET_RX_BUSY_POLL 985 case SO_BUSY_POLL: 986 /* allow unprivileged users to decrease the value */ 987 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN)) 988 ret = -EPERM; 989 else { 990 if (val < 0) 991 ret = -EINVAL; 992 else 993 sk->sk_ll_usec = val; 994 } 995 break; 996 #endif 997 998 case SO_MAX_PACING_RATE: 999 sk->sk_max_pacing_rate = val; 1000 sk->sk_pacing_rate = min(sk->sk_pacing_rate, 1001 sk->sk_max_pacing_rate); 1002 break; 1003 1004 case SO_INCOMING_CPU: 1005 sk->sk_incoming_cpu = val; 1006 break; 1007 1008 case SO_CNX_ADVICE: 1009 if (val == 1) 1010 dst_negative_advice(sk); 1011 break; 1012 default: 1013 ret = -ENOPROTOOPT; 1014 break; 1015 } 1016 release_sock(sk); 1017 return ret; 1018 } 1019 EXPORT_SYMBOL(sock_setsockopt); 1020 1021 1022 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1023 struct ucred *ucred) 1024 { 1025 ucred->pid = pid_vnr(pid); 1026 ucred->uid = ucred->gid = -1; 1027 if (cred) { 1028 struct user_namespace *current_ns = current_user_ns(); 1029 1030 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1031 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1032 } 1033 } 1034 1035 int sock_getsockopt(struct socket *sock, int level, int optname, 1036 char __user *optval, int __user *optlen) 1037 { 1038 struct sock *sk = sock->sk; 1039 1040 union { 1041 int val; 1042 struct linger ling; 1043 struct timeval tm; 1044 } v; 1045 1046 int lv = sizeof(int); 1047 int len; 1048 1049 if (get_user(len, optlen)) 1050 return -EFAULT; 1051 if (len < 0) 1052 return -EINVAL; 1053 1054 memset(&v, 0, sizeof(v)); 1055 1056 switch (optname) { 1057 case SO_DEBUG: 1058 v.val = sock_flag(sk, SOCK_DBG); 1059 break; 1060 1061 case SO_DONTROUTE: 1062 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1063 break; 1064 1065 case SO_BROADCAST: 1066 v.val = sock_flag(sk, SOCK_BROADCAST); 1067 break; 1068 1069 case SO_SNDBUF: 1070 v.val = sk->sk_sndbuf; 1071 break; 1072 1073 case SO_RCVBUF: 1074 v.val = sk->sk_rcvbuf; 1075 break; 1076 1077 case SO_REUSEADDR: 1078 v.val = sk->sk_reuse; 1079 break; 1080 1081 case SO_REUSEPORT: 1082 v.val = sk->sk_reuseport; 1083 break; 1084 1085 case SO_KEEPALIVE: 1086 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1087 break; 1088 1089 case SO_TYPE: 1090 v.val = sk->sk_type; 1091 break; 1092 1093 case SO_PROTOCOL: 1094 v.val = sk->sk_protocol; 1095 break; 1096 1097 case SO_DOMAIN: 1098 v.val = sk->sk_family; 1099 break; 1100 1101 case SO_ERROR: 1102 v.val = -sock_error(sk); 1103 if (v.val == 0) 1104 v.val = xchg(&sk->sk_err_soft, 0); 1105 break; 1106 1107 case SO_OOBINLINE: 1108 v.val = sock_flag(sk, SOCK_URGINLINE); 1109 break; 1110 1111 case SO_NO_CHECK: 1112 v.val = sk->sk_no_check_tx; 1113 break; 1114 1115 case SO_PRIORITY: 1116 v.val = sk->sk_priority; 1117 break; 1118 1119 case SO_LINGER: 1120 lv = sizeof(v.ling); 1121 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1122 v.ling.l_linger = sk->sk_lingertime / HZ; 1123 break; 1124 1125 case SO_BSDCOMPAT: 1126 sock_warn_obsolete_bsdism("getsockopt"); 1127 break; 1128 1129 case SO_TIMESTAMP: 1130 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1131 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1132 break; 1133 1134 case SO_TIMESTAMPNS: 1135 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS); 1136 break; 1137 1138 case SO_TIMESTAMPING: 1139 v.val = sk->sk_tsflags; 1140 break; 1141 1142 case SO_RCVTIMEO: 1143 lv = sizeof(struct timeval); 1144 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) { 1145 v.tm.tv_sec = 0; 1146 v.tm.tv_usec = 0; 1147 } else { 1148 v.tm.tv_sec = sk->sk_rcvtimeo / HZ; 1149 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ; 1150 } 1151 break; 1152 1153 case SO_SNDTIMEO: 1154 lv = sizeof(struct timeval); 1155 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) { 1156 v.tm.tv_sec = 0; 1157 v.tm.tv_usec = 0; 1158 } else { 1159 v.tm.tv_sec = sk->sk_sndtimeo / HZ; 1160 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ; 1161 } 1162 break; 1163 1164 case SO_RCVLOWAT: 1165 v.val = sk->sk_rcvlowat; 1166 break; 1167 1168 case SO_SNDLOWAT: 1169 v.val = 1; 1170 break; 1171 1172 case SO_PASSCRED: 1173 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1174 break; 1175 1176 case SO_PEERCRED: 1177 { 1178 struct ucred peercred; 1179 if (len > sizeof(peercred)) 1180 len = sizeof(peercred); 1181 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1182 if (copy_to_user(optval, &peercred, len)) 1183 return -EFAULT; 1184 goto lenout; 1185 } 1186 1187 case SO_PEERNAME: 1188 { 1189 char address[128]; 1190 1191 if (sock->ops->getname(sock, (struct sockaddr *)address, &lv, 2)) 1192 return -ENOTCONN; 1193 if (lv < len) 1194 return -EINVAL; 1195 if (copy_to_user(optval, address, len)) 1196 return -EFAULT; 1197 goto lenout; 1198 } 1199 1200 /* Dubious BSD thing... Probably nobody even uses it, but 1201 * the UNIX standard wants it for whatever reason... -DaveM 1202 */ 1203 case SO_ACCEPTCONN: 1204 v.val = sk->sk_state == TCP_LISTEN; 1205 break; 1206 1207 case SO_PASSSEC: 1208 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1209 break; 1210 1211 case SO_PEERSEC: 1212 return security_socket_getpeersec_stream(sock, optval, optlen, len); 1213 1214 case SO_MARK: 1215 v.val = sk->sk_mark; 1216 break; 1217 1218 case SO_RXQ_OVFL: 1219 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1220 break; 1221 1222 case SO_WIFI_STATUS: 1223 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1224 break; 1225 1226 case SO_PEEK_OFF: 1227 if (!sock->ops->set_peek_off) 1228 return -EOPNOTSUPP; 1229 1230 v.val = sk->sk_peek_off; 1231 break; 1232 case SO_NOFCS: 1233 v.val = sock_flag(sk, SOCK_NOFCS); 1234 break; 1235 1236 case SO_BINDTODEVICE: 1237 return sock_getbindtodevice(sk, optval, optlen, len); 1238 1239 case SO_GET_FILTER: 1240 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len); 1241 if (len < 0) 1242 return len; 1243 1244 goto lenout; 1245 1246 case SO_LOCK_FILTER: 1247 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1248 break; 1249 1250 case SO_BPF_EXTENSIONS: 1251 v.val = bpf_tell_extensions(); 1252 break; 1253 1254 case SO_SELECT_ERR_QUEUE: 1255 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1256 break; 1257 1258 #ifdef CONFIG_NET_RX_BUSY_POLL 1259 case SO_BUSY_POLL: 1260 v.val = sk->sk_ll_usec; 1261 break; 1262 #endif 1263 1264 case SO_MAX_PACING_RATE: 1265 v.val = sk->sk_max_pacing_rate; 1266 break; 1267 1268 case SO_INCOMING_CPU: 1269 v.val = sk->sk_incoming_cpu; 1270 break; 1271 1272 default: 1273 /* We implement the SO_SNDLOWAT etc to not be settable 1274 * (1003.1g 7). 1275 */ 1276 return -ENOPROTOOPT; 1277 } 1278 1279 if (len > lv) 1280 len = lv; 1281 if (copy_to_user(optval, &v, len)) 1282 return -EFAULT; 1283 lenout: 1284 if (put_user(len, optlen)) 1285 return -EFAULT; 1286 return 0; 1287 } 1288 1289 /* 1290 * Initialize an sk_lock. 1291 * 1292 * (We also register the sk_lock with the lock validator.) 1293 */ 1294 static inline void sock_lock_init(struct sock *sk) 1295 { 1296 sock_lock_init_class_and_name(sk, 1297 af_family_slock_key_strings[sk->sk_family], 1298 af_family_slock_keys + sk->sk_family, 1299 af_family_key_strings[sk->sk_family], 1300 af_family_keys + sk->sk_family); 1301 } 1302 1303 /* 1304 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 1305 * even temporarly, because of RCU lookups. sk_node should also be left as is. 1306 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 1307 */ 1308 static void sock_copy(struct sock *nsk, const struct sock *osk) 1309 { 1310 #ifdef CONFIG_SECURITY_NETWORK 1311 void *sptr = nsk->sk_security; 1312 #endif 1313 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 1314 1315 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 1316 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 1317 1318 #ifdef CONFIG_SECURITY_NETWORK 1319 nsk->sk_security = sptr; 1320 security_sk_clone(osk, nsk); 1321 #endif 1322 } 1323 1324 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 1325 int family) 1326 { 1327 struct sock *sk; 1328 struct kmem_cache *slab; 1329 1330 slab = prot->slab; 1331 if (slab != NULL) { 1332 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 1333 if (!sk) 1334 return sk; 1335 if (priority & __GFP_ZERO) 1336 sk_prot_clear_nulls(sk, prot->obj_size); 1337 } else 1338 sk = kmalloc(prot->obj_size, priority); 1339 1340 if (sk != NULL) { 1341 kmemcheck_annotate_bitfield(sk, flags); 1342 1343 if (security_sk_alloc(sk, family, priority)) 1344 goto out_free; 1345 1346 if (!try_module_get(prot->owner)) 1347 goto out_free_sec; 1348 sk_tx_queue_clear(sk); 1349 } 1350 1351 return sk; 1352 1353 out_free_sec: 1354 security_sk_free(sk); 1355 out_free: 1356 if (slab != NULL) 1357 kmem_cache_free(slab, sk); 1358 else 1359 kfree(sk); 1360 return NULL; 1361 } 1362 1363 static void sk_prot_free(struct proto *prot, struct sock *sk) 1364 { 1365 struct kmem_cache *slab; 1366 struct module *owner; 1367 1368 owner = prot->owner; 1369 slab = prot->slab; 1370 1371 cgroup_sk_free(&sk->sk_cgrp_data); 1372 mem_cgroup_sk_free(sk); 1373 security_sk_free(sk); 1374 if (slab != NULL) 1375 kmem_cache_free(slab, sk); 1376 else 1377 kfree(sk); 1378 module_put(owner); 1379 } 1380 1381 /** 1382 * sk_alloc - All socket objects are allocated here 1383 * @net: the applicable net namespace 1384 * @family: protocol family 1385 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1386 * @prot: struct proto associated with this new sock instance 1387 * @kern: is this to be a kernel socket? 1388 */ 1389 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 1390 struct proto *prot, int kern) 1391 { 1392 struct sock *sk; 1393 1394 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 1395 if (sk) { 1396 sk->sk_family = family; 1397 /* 1398 * See comment in struct sock definition to understand 1399 * why we need sk_prot_creator -acme 1400 */ 1401 sk->sk_prot = sk->sk_prot_creator = prot; 1402 sock_lock_init(sk); 1403 sk->sk_net_refcnt = kern ? 0 : 1; 1404 if (likely(sk->sk_net_refcnt)) 1405 get_net(net); 1406 sock_net_set(sk, net); 1407 atomic_set(&sk->sk_wmem_alloc, 1); 1408 1409 mem_cgroup_sk_alloc(sk); 1410 cgroup_sk_alloc(&sk->sk_cgrp_data); 1411 sock_update_classid(&sk->sk_cgrp_data); 1412 sock_update_netprioidx(&sk->sk_cgrp_data); 1413 } 1414 1415 return sk; 1416 } 1417 EXPORT_SYMBOL(sk_alloc); 1418 1419 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 1420 * grace period. This is the case for UDP sockets and TCP listeners. 1421 */ 1422 static void __sk_destruct(struct rcu_head *head) 1423 { 1424 struct sock *sk = container_of(head, struct sock, sk_rcu); 1425 struct sk_filter *filter; 1426 1427 if (sk->sk_destruct) 1428 sk->sk_destruct(sk); 1429 1430 filter = rcu_dereference_check(sk->sk_filter, 1431 atomic_read(&sk->sk_wmem_alloc) == 0); 1432 if (filter) { 1433 sk_filter_uncharge(sk, filter); 1434 RCU_INIT_POINTER(sk->sk_filter, NULL); 1435 } 1436 if (rcu_access_pointer(sk->sk_reuseport_cb)) 1437 reuseport_detach_sock(sk); 1438 1439 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 1440 1441 if (atomic_read(&sk->sk_omem_alloc)) 1442 pr_debug("%s: optmem leakage (%d bytes) detected\n", 1443 __func__, atomic_read(&sk->sk_omem_alloc)); 1444 1445 if (sk->sk_peer_cred) 1446 put_cred(sk->sk_peer_cred); 1447 put_pid(sk->sk_peer_pid); 1448 if (likely(sk->sk_net_refcnt)) 1449 put_net(sock_net(sk)); 1450 sk_prot_free(sk->sk_prot_creator, sk); 1451 } 1452 1453 void sk_destruct(struct sock *sk) 1454 { 1455 if (sock_flag(sk, SOCK_RCU_FREE)) 1456 call_rcu(&sk->sk_rcu, __sk_destruct); 1457 else 1458 __sk_destruct(&sk->sk_rcu); 1459 } 1460 1461 static void __sk_free(struct sock *sk) 1462 { 1463 if (unlikely(sock_diag_has_destroy_listeners(sk) && sk->sk_net_refcnt)) 1464 sock_diag_broadcast_destroy(sk); 1465 else 1466 sk_destruct(sk); 1467 } 1468 1469 void sk_free(struct sock *sk) 1470 { 1471 /* 1472 * We subtract one from sk_wmem_alloc and can know if 1473 * some packets are still in some tx queue. 1474 * If not null, sock_wfree() will call __sk_free(sk) later 1475 */ 1476 if (atomic_dec_and_test(&sk->sk_wmem_alloc)) 1477 __sk_free(sk); 1478 } 1479 EXPORT_SYMBOL(sk_free); 1480 1481 /** 1482 * sk_clone_lock - clone a socket, and lock its clone 1483 * @sk: the socket to clone 1484 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1485 * 1486 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1487 */ 1488 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 1489 { 1490 struct sock *newsk; 1491 bool is_charged = true; 1492 1493 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family); 1494 if (newsk != NULL) { 1495 struct sk_filter *filter; 1496 1497 sock_copy(newsk, sk); 1498 1499 /* SANITY */ 1500 if (likely(newsk->sk_net_refcnt)) 1501 get_net(sock_net(newsk)); 1502 sk_node_init(&newsk->sk_node); 1503 sock_lock_init(newsk); 1504 bh_lock_sock(newsk); 1505 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 1506 newsk->sk_backlog.len = 0; 1507 1508 atomic_set(&newsk->sk_rmem_alloc, 0); 1509 /* 1510 * sk_wmem_alloc set to one (see sk_free() and sock_wfree()) 1511 */ 1512 atomic_set(&newsk->sk_wmem_alloc, 1); 1513 atomic_set(&newsk->sk_omem_alloc, 0); 1514 skb_queue_head_init(&newsk->sk_receive_queue); 1515 skb_queue_head_init(&newsk->sk_write_queue); 1516 1517 rwlock_init(&newsk->sk_callback_lock); 1518 lockdep_set_class_and_name(&newsk->sk_callback_lock, 1519 af_callback_keys + newsk->sk_family, 1520 af_family_clock_key_strings[newsk->sk_family]); 1521 1522 newsk->sk_dst_cache = NULL; 1523 newsk->sk_dst_pending_confirm = 0; 1524 newsk->sk_wmem_queued = 0; 1525 newsk->sk_forward_alloc = 0; 1526 atomic_set(&newsk->sk_drops, 0); 1527 newsk->sk_send_head = NULL; 1528 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 1529 1530 sock_reset_flag(newsk, SOCK_DONE); 1531 skb_queue_head_init(&newsk->sk_error_queue); 1532 1533 filter = rcu_dereference_protected(newsk->sk_filter, 1); 1534 if (filter != NULL) 1535 /* though it's an empty new sock, the charging may fail 1536 * if sysctl_optmem_max was changed between creation of 1537 * original socket and cloning 1538 */ 1539 is_charged = sk_filter_charge(newsk, filter); 1540 1541 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 1542 /* It is still raw copy of parent, so invalidate 1543 * destructor and make plain sk_free() */ 1544 newsk->sk_destruct = NULL; 1545 bh_unlock_sock(newsk); 1546 sk_free(newsk); 1547 newsk = NULL; 1548 goto out; 1549 } 1550 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 1551 1552 newsk->sk_err = 0; 1553 newsk->sk_err_soft = 0; 1554 newsk->sk_priority = 0; 1555 newsk->sk_incoming_cpu = raw_smp_processor_id(); 1556 atomic64_set(&newsk->sk_cookie, 0); 1557 1558 mem_cgroup_sk_alloc(newsk); 1559 cgroup_sk_alloc(&newsk->sk_cgrp_data); 1560 1561 /* 1562 * Before updating sk_refcnt, we must commit prior changes to memory 1563 * (Documentation/RCU/rculist_nulls.txt for details) 1564 */ 1565 smp_wmb(); 1566 atomic_set(&newsk->sk_refcnt, 2); 1567 1568 /* 1569 * Increment the counter in the same struct proto as the master 1570 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that 1571 * is the same as sk->sk_prot->socks, as this field was copied 1572 * with memcpy). 1573 * 1574 * This _changes_ the previous behaviour, where 1575 * tcp_create_openreq_child always was incrementing the 1576 * equivalent to tcp_prot->socks (inet_sock_nr), so this have 1577 * to be taken into account in all callers. -acme 1578 */ 1579 sk_refcnt_debug_inc(newsk); 1580 sk_set_socket(newsk, NULL); 1581 newsk->sk_wq = NULL; 1582 1583 if (newsk->sk_prot->sockets_allocated) 1584 sk_sockets_allocated_inc(newsk); 1585 1586 if (sock_needs_netstamp(sk) && 1587 newsk->sk_flags & SK_FLAGS_TIMESTAMP) 1588 net_enable_timestamp(); 1589 } 1590 out: 1591 return newsk; 1592 } 1593 EXPORT_SYMBOL_GPL(sk_clone_lock); 1594 1595 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 1596 { 1597 u32 max_segs = 1; 1598 1599 sk_dst_set(sk, dst); 1600 sk->sk_route_caps = dst->dev->features; 1601 if (sk->sk_route_caps & NETIF_F_GSO) 1602 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 1603 sk->sk_route_caps &= ~sk->sk_route_nocaps; 1604 if (sk_can_gso(sk)) { 1605 if (dst->header_len) { 1606 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 1607 } else { 1608 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 1609 sk->sk_gso_max_size = dst->dev->gso_max_size; 1610 max_segs = max_t(u32, dst->dev->gso_max_segs, 1); 1611 } 1612 } 1613 sk->sk_gso_max_segs = max_segs; 1614 } 1615 EXPORT_SYMBOL_GPL(sk_setup_caps); 1616 1617 /* 1618 * Simple resource managers for sockets. 1619 */ 1620 1621 1622 /* 1623 * Write buffer destructor automatically called from kfree_skb. 1624 */ 1625 void sock_wfree(struct sk_buff *skb) 1626 { 1627 struct sock *sk = skb->sk; 1628 unsigned int len = skb->truesize; 1629 1630 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 1631 /* 1632 * Keep a reference on sk_wmem_alloc, this will be released 1633 * after sk_write_space() call 1634 */ 1635 atomic_sub(len - 1, &sk->sk_wmem_alloc); 1636 sk->sk_write_space(sk); 1637 len = 1; 1638 } 1639 /* 1640 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 1641 * could not do because of in-flight packets 1642 */ 1643 if (atomic_sub_and_test(len, &sk->sk_wmem_alloc)) 1644 __sk_free(sk); 1645 } 1646 EXPORT_SYMBOL(sock_wfree); 1647 1648 /* This variant of sock_wfree() is used by TCP, 1649 * since it sets SOCK_USE_WRITE_QUEUE. 1650 */ 1651 void __sock_wfree(struct sk_buff *skb) 1652 { 1653 struct sock *sk = skb->sk; 1654 1655 if (atomic_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 1656 __sk_free(sk); 1657 } 1658 1659 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 1660 { 1661 skb_orphan(skb); 1662 skb->sk = sk; 1663 #ifdef CONFIG_INET 1664 if (unlikely(!sk_fullsock(sk))) { 1665 skb->destructor = sock_edemux; 1666 sock_hold(sk); 1667 return; 1668 } 1669 #endif 1670 skb->destructor = sock_wfree; 1671 skb_set_hash_from_sk(skb, sk); 1672 /* 1673 * We used to take a refcount on sk, but following operation 1674 * is enough to guarantee sk_free() wont free this sock until 1675 * all in-flight packets are completed 1676 */ 1677 atomic_add(skb->truesize, &sk->sk_wmem_alloc); 1678 } 1679 EXPORT_SYMBOL(skb_set_owner_w); 1680 1681 /* This helper is used by netem, as it can hold packets in its 1682 * delay queue. We want to allow the owner socket to send more 1683 * packets, as if they were already TX completed by a typical driver. 1684 * But we also want to keep skb->sk set because some packet schedulers 1685 * rely on it (sch_fq for example). So we set skb->truesize to a small 1686 * amount (1) and decrease sk_wmem_alloc accordingly. 1687 */ 1688 void skb_orphan_partial(struct sk_buff *skb) 1689 { 1690 /* If this skb is a TCP pure ACK or already went here, 1691 * we have nothing to do. 2 is already a very small truesize. 1692 */ 1693 if (skb->truesize <= 2) 1694 return; 1695 1696 /* TCP stack sets skb->ooo_okay based on sk_wmem_alloc, 1697 * so we do not completely orphan skb, but transfert all 1698 * accounted bytes but one, to avoid unexpected reorders. 1699 */ 1700 if (skb->destructor == sock_wfree 1701 #ifdef CONFIG_INET 1702 || skb->destructor == tcp_wfree 1703 #endif 1704 ) { 1705 atomic_sub(skb->truesize - 1, &skb->sk->sk_wmem_alloc); 1706 skb->truesize = 1; 1707 } else { 1708 skb_orphan(skb); 1709 } 1710 } 1711 EXPORT_SYMBOL(skb_orphan_partial); 1712 1713 /* 1714 * Read buffer destructor automatically called from kfree_skb. 1715 */ 1716 void sock_rfree(struct sk_buff *skb) 1717 { 1718 struct sock *sk = skb->sk; 1719 unsigned int len = skb->truesize; 1720 1721 atomic_sub(len, &sk->sk_rmem_alloc); 1722 sk_mem_uncharge(sk, len); 1723 } 1724 EXPORT_SYMBOL(sock_rfree); 1725 1726 /* 1727 * Buffer destructor for skbs that are not used directly in read or write 1728 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 1729 */ 1730 void sock_efree(struct sk_buff *skb) 1731 { 1732 sock_put(skb->sk); 1733 } 1734 EXPORT_SYMBOL(sock_efree); 1735 1736 kuid_t sock_i_uid(struct sock *sk) 1737 { 1738 kuid_t uid; 1739 1740 read_lock_bh(&sk->sk_callback_lock); 1741 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 1742 read_unlock_bh(&sk->sk_callback_lock); 1743 return uid; 1744 } 1745 EXPORT_SYMBOL(sock_i_uid); 1746 1747 unsigned long sock_i_ino(struct sock *sk) 1748 { 1749 unsigned long ino; 1750 1751 read_lock_bh(&sk->sk_callback_lock); 1752 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 1753 read_unlock_bh(&sk->sk_callback_lock); 1754 return ino; 1755 } 1756 EXPORT_SYMBOL(sock_i_ino); 1757 1758 /* 1759 * Allocate a skb from the socket's send buffer. 1760 */ 1761 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 1762 gfp_t priority) 1763 { 1764 if (force || atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) { 1765 struct sk_buff *skb = alloc_skb(size, priority); 1766 if (skb) { 1767 skb_set_owner_w(skb, sk); 1768 return skb; 1769 } 1770 } 1771 return NULL; 1772 } 1773 EXPORT_SYMBOL(sock_wmalloc); 1774 1775 /* 1776 * Allocate a memory block from the socket's option memory buffer. 1777 */ 1778 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 1779 { 1780 if ((unsigned int)size <= sysctl_optmem_max && 1781 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { 1782 void *mem; 1783 /* First do the add, to avoid the race if kmalloc 1784 * might sleep. 1785 */ 1786 atomic_add(size, &sk->sk_omem_alloc); 1787 mem = kmalloc(size, priority); 1788 if (mem) 1789 return mem; 1790 atomic_sub(size, &sk->sk_omem_alloc); 1791 } 1792 return NULL; 1793 } 1794 EXPORT_SYMBOL(sock_kmalloc); 1795 1796 /* Free an option memory block. Note, we actually want the inline 1797 * here as this allows gcc to detect the nullify and fold away the 1798 * condition entirely. 1799 */ 1800 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 1801 const bool nullify) 1802 { 1803 if (WARN_ON_ONCE(!mem)) 1804 return; 1805 if (nullify) 1806 kzfree(mem); 1807 else 1808 kfree(mem); 1809 atomic_sub(size, &sk->sk_omem_alloc); 1810 } 1811 1812 void sock_kfree_s(struct sock *sk, void *mem, int size) 1813 { 1814 __sock_kfree_s(sk, mem, size, false); 1815 } 1816 EXPORT_SYMBOL(sock_kfree_s); 1817 1818 void sock_kzfree_s(struct sock *sk, void *mem, int size) 1819 { 1820 __sock_kfree_s(sk, mem, size, true); 1821 } 1822 EXPORT_SYMBOL(sock_kzfree_s); 1823 1824 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 1825 I think, these locks should be removed for datagram sockets. 1826 */ 1827 static long sock_wait_for_wmem(struct sock *sk, long timeo) 1828 { 1829 DEFINE_WAIT(wait); 1830 1831 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1832 for (;;) { 1833 if (!timeo) 1834 break; 1835 if (signal_pending(current)) 1836 break; 1837 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1838 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 1839 if (atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) 1840 break; 1841 if (sk->sk_shutdown & SEND_SHUTDOWN) 1842 break; 1843 if (sk->sk_err) 1844 break; 1845 timeo = schedule_timeout(timeo); 1846 } 1847 finish_wait(sk_sleep(sk), &wait); 1848 return timeo; 1849 } 1850 1851 1852 /* 1853 * Generic send/receive buffer handlers 1854 */ 1855 1856 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 1857 unsigned long data_len, int noblock, 1858 int *errcode, int max_page_order) 1859 { 1860 struct sk_buff *skb; 1861 long timeo; 1862 int err; 1863 1864 timeo = sock_sndtimeo(sk, noblock); 1865 for (;;) { 1866 err = sock_error(sk); 1867 if (err != 0) 1868 goto failure; 1869 1870 err = -EPIPE; 1871 if (sk->sk_shutdown & SEND_SHUTDOWN) 1872 goto failure; 1873 1874 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf) 1875 break; 1876 1877 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 1878 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1879 err = -EAGAIN; 1880 if (!timeo) 1881 goto failure; 1882 if (signal_pending(current)) 1883 goto interrupted; 1884 timeo = sock_wait_for_wmem(sk, timeo); 1885 } 1886 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 1887 errcode, sk->sk_allocation); 1888 if (skb) 1889 skb_set_owner_w(skb, sk); 1890 return skb; 1891 1892 interrupted: 1893 err = sock_intr_errno(timeo); 1894 failure: 1895 *errcode = err; 1896 return NULL; 1897 } 1898 EXPORT_SYMBOL(sock_alloc_send_pskb); 1899 1900 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size, 1901 int noblock, int *errcode) 1902 { 1903 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0); 1904 } 1905 EXPORT_SYMBOL(sock_alloc_send_skb); 1906 1907 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, 1908 struct sockcm_cookie *sockc) 1909 { 1910 u32 tsflags; 1911 1912 switch (cmsg->cmsg_type) { 1913 case SO_MARK: 1914 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 1915 return -EPERM; 1916 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 1917 return -EINVAL; 1918 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 1919 break; 1920 case SO_TIMESTAMPING: 1921 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 1922 return -EINVAL; 1923 1924 tsflags = *(u32 *)CMSG_DATA(cmsg); 1925 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 1926 return -EINVAL; 1927 1928 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 1929 sockc->tsflags |= tsflags; 1930 break; 1931 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 1932 case SCM_RIGHTS: 1933 case SCM_CREDENTIALS: 1934 break; 1935 default: 1936 return -EINVAL; 1937 } 1938 return 0; 1939 } 1940 EXPORT_SYMBOL(__sock_cmsg_send); 1941 1942 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 1943 struct sockcm_cookie *sockc) 1944 { 1945 struct cmsghdr *cmsg; 1946 int ret; 1947 1948 for_each_cmsghdr(cmsg, msg) { 1949 if (!CMSG_OK(msg, cmsg)) 1950 return -EINVAL; 1951 if (cmsg->cmsg_level != SOL_SOCKET) 1952 continue; 1953 ret = __sock_cmsg_send(sk, msg, cmsg, sockc); 1954 if (ret) 1955 return ret; 1956 } 1957 return 0; 1958 } 1959 EXPORT_SYMBOL(sock_cmsg_send); 1960 1961 /* On 32bit arches, an skb frag is limited to 2^15 */ 1962 #define SKB_FRAG_PAGE_ORDER get_order(32768) 1963 1964 /** 1965 * skb_page_frag_refill - check that a page_frag contains enough room 1966 * @sz: minimum size of the fragment we want to get 1967 * @pfrag: pointer to page_frag 1968 * @gfp: priority for memory allocation 1969 * 1970 * Note: While this allocator tries to use high order pages, there is 1971 * no guarantee that allocations succeed. Therefore, @sz MUST be 1972 * less or equal than PAGE_SIZE. 1973 */ 1974 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 1975 { 1976 if (pfrag->page) { 1977 if (page_ref_count(pfrag->page) == 1) { 1978 pfrag->offset = 0; 1979 return true; 1980 } 1981 if (pfrag->offset + sz <= pfrag->size) 1982 return true; 1983 put_page(pfrag->page); 1984 } 1985 1986 pfrag->offset = 0; 1987 if (SKB_FRAG_PAGE_ORDER) { 1988 /* Avoid direct reclaim but allow kswapd to wake */ 1989 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 1990 __GFP_COMP | __GFP_NOWARN | 1991 __GFP_NORETRY, 1992 SKB_FRAG_PAGE_ORDER); 1993 if (likely(pfrag->page)) { 1994 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 1995 return true; 1996 } 1997 } 1998 pfrag->page = alloc_page(gfp); 1999 if (likely(pfrag->page)) { 2000 pfrag->size = PAGE_SIZE; 2001 return true; 2002 } 2003 return false; 2004 } 2005 EXPORT_SYMBOL(skb_page_frag_refill); 2006 2007 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2008 { 2009 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2010 return true; 2011 2012 sk_enter_memory_pressure(sk); 2013 sk_stream_moderate_sndbuf(sk); 2014 return false; 2015 } 2016 EXPORT_SYMBOL(sk_page_frag_refill); 2017 2018 static void __lock_sock(struct sock *sk) 2019 __releases(&sk->sk_lock.slock) 2020 __acquires(&sk->sk_lock.slock) 2021 { 2022 DEFINE_WAIT(wait); 2023 2024 for (;;) { 2025 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2026 TASK_UNINTERRUPTIBLE); 2027 spin_unlock_bh(&sk->sk_lock.slock); 2028 schedule(); 2029 spin_lock_bh(&sk->sk_lock.slock); 2030 if (!sock_owned_by_user(sk)) 2031 break; 2032 } 2033 finish_wait(&sk->sk_lock.wq, &wait); 2034 } 2035 2036 static void __release_sock(struct sock *sk) 2037 __releases(&sk->sk_lock.slock) 2038 __acquires(&sk->sk_lock.slock) 2039 { 2040 struct sk_buff *skb, *next; 2041 2042 while ((skb = sk->sk_backlog.head) != NULL) { 2043 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2044 2045 spin_unlock_bh(&sk->sk_lock.slock); 2046 2047 do { 2048 next = skb->next; 2049 prefetch(next); 2050 WARN_ON_ONCE(skb_dst_is_noref(skb)); 2051 skb->next = NULL; 2052 sk_backlog_rcv(sk, skb); 2053 2054 cond_resched(); 2055 2056 skb = next; 2057 } while (skb != NULL); 2058 2059 spin_lock_bh(&sk->sk_lock.slock); 2060 } 2061 2062 /* 2063 * Doing the zeroing here guarantee we can not loop forever 2064 * while a wild producer attempts to flood us. 2065 */ 2066 sk->sk_backlog.len = 0; 2067 } 2068 2069 void __sk_flush_backlog(struct sock *sk) 2070 { 2071 spin_lock_bh(&sk->sk_lock.slock); 2072 __release_sock(sk); 2073 spin_unlock_bh(&sk->sk_lock.slock); 2074 } 2075 2076 /** 2077 * sk_wait_data - wait for data to arrive at sk_receive_queue 2078 * @sk: sock to wait on 2079 * @timeo: for how long 2080 * @skb: last skb seen on sk_receive_queue 2081 * 2082 * Now socket state including sk->sk_err is changed only under lock, 2083 * hence we may omit checks after joining wait queue. 2084 * We check receive queue before schedule() only as optimization; 2085 * it is very likely that release_sock() added new data. 2086 */ 2087 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 2088 { 2089 DEFINE_WAIT_FUNC(wait, woken_wake_function); 2090 int rc; 2091 2092 add_wait_queue(sk_sleep(sk), &wait); 2093 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2094 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 2095 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2096 remove_wait_queue(sk_sleep(sk), &wait); 2097 return rc; 2098 } 2099 EXPORT_SYMBOL(sk_wait_data); 2100 2101 /** 2102 * __sk_mem_raise_allocated - increase memory_allocated 2103 * @sk: socket 2104 * @size: memory size to allocate 2105 * @amt: pages to allocate 2106 * @kind: allocation type 2107 * 2108 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc 2109 */ 2110 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 2111 { 2112 struct proto *prot = sk->sk_prot; 2113 long allocated = sk_memory_allocated_add(sk, amt); 2114 2115 if (mem_cgroup_sockets_enabled && sk->sk_memcg && 2116 !mem_cgroup_charge_skmem(sk->sk_memcg, amt)) 2117 goto suppress_allocation; 2118 2119 /* Under limit. */ 2120 if (allocated <= sk_prot_mem_limits(sk, 0)) { 2121 sk_leave_memory_pressure(sk); 2122 return 1; 2123 } 2124 2125 /* Under pressure. */ 2126 if (allocated > sk_prot_mem_limits(sk, 1)) 2127 sk_enter_memory_pressure(sk); 2128 2129 /* Over hard limit. */ 2130 if (allocated > sk_prot_mem_limits(sk, 2)) 2131 goto suppress_allocation; 2132 2133 /* guarantee minimum buffer size under pressure */ 2134 if (kind == SK_MEM_RECV) { 2135 if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0]) 2136 return 1; 2137 2138 } else { /* SK_MEM_SEND */ 2139 if (sk->sk_type == SOCK_STREAM) { 2140 if (sk->sk_wmem_queued < prot->sysctl_wmem[0]) 2141 return 1; 2142 } else if (atomic_read(&sk->sk_wmem_alloc) < 2143 prot->sysctl_wmem[0]) 2144 return 1; 2145 } 2146 2147 if (sk_has_memory_pressure(sk)) { 2148 int alloc; 2149 2150 if (!sk_under_memory_pressure(sk)) 2151 return 1; 2152 alloc = sk_sockets_allocated_read_positive(sk); 2153 if (sk_prot_mem_limits(sk, 2) > alloc * 2154 sk_mem_pages(sk->sk_wmem_queued + 2155 atomic_read(&sk->sk_rmem_alloc) + 2156 sk->sk_forward_alloc)) 2157 return 1; 2158 } 2159 2160 suppress_allocation: 2161 2162 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 2163 sk_stream_moderate_sndbuf(sk); 2164 2165 /* Fail only if socket is _under_ its sndbuf. 2166 * In this case we cannot block, so that we have to fail. 2167 */ 2168 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) 2169 return 1; 2170 } 2171 2172 trace_sock_exceed_buf_limit(sk, prot, allocated); 2173 2174 sk_memory_allocated_sub(sk, amt); 2175 2176 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2177 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt); 2178 2179 return 0; 2180 } 2181 EXPORT_SYMBOL(__sk_mem_raise_allocated); 2182 2183 /** 2184 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 2185 * @sk: socket 2186 * @size: memory size to allocate 2187 * @kind: allocation type 2188 * 2189 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 2190 * rmem allocation. This function assumes that protocols which have 2191 * memory_pressure use sk_wmem_queued as write buffer accounting. 2192 */ 2193 int __sk_mem_schedule(struct sock *sk, int size, int kind) 2194 { 2195 int ret, amt = sk_mem_pages(size); 2196 2197 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT; 2198 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 2199 if (!ret) 2200 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT; 2201 return ret; 2202 } 2203 EXPORT_SYMBOL(__sk_mem_schedule); 2204 2205 /** 2206 * __sk_mem_reduce_allocated - reclaim memory_allocated 2207 * @sk: socket 2208 * @amount: number of quanta 2209 * 2210 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 2211 */ 2212 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 2213 { 2214 sk_memory_allocated_sub(sk, amount); 2215 2216 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2217 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 2218 2219 if (sk_under_memory_pressure(sk) && 2220 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 2221 sk_leave_memory_pressure(sk); 2222 } 2223 EXPORT_SYMBOL(__sk_mem_reduce_allocated); 2224 2225 /** 2226 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 2227 * @sk: socket 2228 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple) 2229 */ 2230 void __sk_mem_reclaim(struct sock *sk, int amount) 2231 { 2232 amount >>= SK_MEM_QUANTUM_SHIFT; 2233 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT; 2234 __sk_mem_reduce_allocated(sk, amount); 2235 } 2236 EXPORT_SYMBOL(__sk_mem_reclaim); 2237 2238 int sk_set_peek_off(struct sock *sk, int val) 2239 { 2240 if (val < 0) 2241 return -EINVAL; 2242 2243 sk->sk_peek_off = val; 2244 return 0; 2245 } 2246 EXPORT_SYMBOL_GPL(sk_set_peek_off); 2247 2248 /* 2249 * Set of default routines for initialising struct proto_ops when 2250 * the protocol does not support a particular function. In certain 2251 * cases where it makes no sense for a protocol to have a "do nothing" 2252 * function, some default processing is provided. 2253 */ 2254 2255 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 2256 { 2257 return -EOPNOTSUPP; 2258 } 2259 EXPORT_SYMBOL(sock_no_bind); 2260 2261 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 2262 int len, int flags) 2263 { 2264 return -EOPNOTSUPP; 2265 } 2266 EXPORT_SYMBOL(sock_no_connect); 2267 2268 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 2269 { 2270 return -EOPNOTSUPP; 2271 } 2272 EXPORT_SYMBOL(sock_no_socketpair); 2273 2274 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags) 2275 { 2276 return -EOPNOTSUPP; 2277 } 2278 EXPORT_SYMBOL(sock_no_accept); 2279 2280 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 2281 int *len, int peer) 2282 { 2283 return -EOPNOTSUPP; 2284 } 2285 EXPORT_SYMBOL(sock_no_getname); 2286 2287 unsigned int sock_no_poll(struct file *file, struct socket *sock, poll_table *pt) 2288 { 2289 return 0; 2290 } 2291 EXPORT_SYMBOL(sock_no_poll); 2292 2293 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 2294 { 2295 return -EOPNOTSUPP; 2296 } 2297 EXPORT_SYMBOL(sock_no_ioctl); 2298 2299 int sock_no_listen(struct socket *sock, int backlog) 2300 { 2301 return -EOPNOTSUPP; 2302 } 2303 EXPORT_SYMBOL(sock_no_listen); 2304 2305 int sock_no_shutdown(struct socket *sock, int how) 2306 { 2307 return -EOPNOTSUPP; 2308 } 2309 EXPORT_SYMBOL(sock_no_shutdown); 2310 2311 int sock_no_setsockopt(struct socket *sock, int level, int optname, 2312 char __user *optval, unsigned int optlen) 2313 { 2314 return -EOPNOTSUPP; 2315 } 2316 EXPORT_SYMBOL(sock_no_setsockopt); 2317 2318 int sock_no_getsockopt(struct socket *sock, int level, int optname, 2319 char __user *optval, int __user *optlen) 2320 { 2321 return -EOPNOTSUPP; 2322 } 2323 EXPORT_SYMBOL(sock_no_getsockopt); 2324 2325 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 2326 { 2327 return -EOPNOTSUPP; 2328 } 2329 EXPORT_SYMBOL(sock_no_sendmsg); 2330 2331 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 2332 int flags) 2333 { 2334 return -EOPNOTSUPP; 2335 } 2336 EXPORT_SYMBOL(sock_no_recvmsg); 2337 2338 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 2339 { 2340 /* Mirror missing mmap method error code */ 2341 return -ENODEV; 2342 } 2343 EXPORT_SYMBOL(sock_no_mmap); 2344 2345 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) 2346 { 2347 ssize_t res; 2348 struct msghdr msg = {.msg_flags = flags}; 2349 struct kvec iov; 2350 char *kaddr = kmap(page); 2351 iov.iov_base = kaddr + offset; 2352 iov.iov_len = size; 2353 res = kernel_sendmsg(sock, &msg, &iov, 1, size); 2354 kunmap(page); 2355 return res; 2356 } 2357 EXPORT_SYMBOL(sock_no_sendpage); 2358 2359 /* 2360 * Default Socket Callbacks 2361 */ 2362 2363 static void sock_def_wakeup(struct sock *sk) 2364 { 2365 struct socket_wq *wq; 2366 2367 rcu_read_lock(); 2368 wq = rcu_dereference(sk->sk_wq); 2369 if (skwq_has_sleeper(wq)) 2370 wake_up_interruptible_all(&wq->wait); 2371 rcu_read_unlock(); 2372 } 2373 2374 static void sock_def_error_report(struct sock *sk) 2375 { 2376 struct socket_wq *wq; 2377 2378 rcu_read_lock(); 2379 wq = rcu_dereference(sk->sk_wq); 2380 if (skwq_has_sleeper(wq)) 2381 wake_up_interruptible_poll(&wq->wait, POLLERR); 2382 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 2383 rcu_read_unlock(); 2384 } 2385 2386 static void sock_def_readable(struct sock *sk) 2387 { 2388 struct socket_wq *wq; 2389 2390 rcu_read_lock(); 2391 wq = rcu_dereference(sk->sk_wq); 2392 if (skwq_has_sleeper(wq)) 2393 wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLPRI | 2394 POLLRDNORM | POLLRDBAND); 2395 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 2396 rcu_read_unlock(); 2397 } 2398 2399 static void sock_def_write_space(struct sock *sk) 2400 { 2401 struct socket_wq *wq; 2402 2403 rcu_read_lock(); 2404 2405 /* Do not wake up a writer until he can make "significant" 2406 * progress. --DaveM 2407 */ 2408 if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) { 2409 wq = rcu_dereference(sk->sk_wq); 2410 if (skwq_has_sleeper(wq)) 2411 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT | 2412 POLLWRNORM | POLLWRBAND); 2413 2414 /* Should agree with poll, otherwise some programs break */ 2415 if (sock_writeable(sk)) 2416 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 2417 } 2418 2419 rcu_read_unlock(); 2420 } 2421 2422 static void sock_def_destruct(struct sock *sk) 2423 { 2424 } 2425 2426 void sk_send_sigurg(struct sock *sk) 2427 { 2428 if (sk->sk_socket && sk->sk_socket->file) 2429 if (send_sigurg(&sk->sk_socket->file->f_owner)) 2430 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 2431 } 2432 EXPORT_SYMBOL(sk_send_sigurg); 2433 2434 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 2435 unsigned long expires) 2436 { 2437 if (!mod_timer(timer, expires)) 2438 sock_hold(sk); 2439 } 2440 EXPORT_SYMBOL(sk_reset_timer); 2441 2442 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 2443 { 2444 if (del_timer(timer)) 2445 __sock_put(sk); 2446 } 2447 EXPORT_SYMBOL(sk_stop_timer); 2448 2449 void sock_init_data(struct socket *sock, struct sock *sk) 2450 { 2451 skb_queue_head_init(&sk->sk_receive_queue); 2452 skb_queue_head_init(&sk->sk_write_queue); 2453 skb_queue_head_init(&sk->sk_error_queue); 2454 2455 sk->sk_send_head = NULL; 2456 2457 init_timer(&sk->sk_timer); 2458 2459 sk->sk_allocation = GFP_KERNEL; 2460 sk->sk_rcvbuf = sysctl_rmem_default; 2461 sk->sk_sndbuf = sysctl_wmem_default; 2462 sk->sk_state = TCP_CLOSE; 2463 sk_set_socket(sk, sock); 2464 2465 sock_set_flag(sk, SOCK_ZAPPED); 2466 2467 if (sock) { 2468 sk->sk_type = sock->type; 2469 sk->sk_wq = sock->wq; 2470 sock->sk = sk; 2471 sk->sk_uid = SOCK_INODE(sock)->i_uid; 2472 } else { 2473 sk->sk_wq = NULL; 2474 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0); 2475 } 2476 2477 rwlock_init(&sk->sk_callback_lock); 2478 lockdep_set_class_and_name(&sk->sk_callback_lock, 2479 af_callback_keys + sk->sk_family, 2480 af_family_clock_key_strings[sk->sk_family]); 2481 2482 sk->sk_state_change = sock_def_wakeup; 2483 sk->sk_data_ready = sock_def_readable; 2484 sk->sk_write_space = sock_def_write_space; 2485 sk->sk_error_report = sock_def_error_report; 2486 sk->sk_destruct = sock_def_destruct; 2487 2488 sk->sk_frag.page = NULL; 2489 sk->sk_frag.offset = 0; 2490 sk->sk_peek_off = -1; 2491 2492 sk->sk_peer_pid = NULL; 2493 sk->sk_peer_cred = NULL; 2494 sk->sk_write_pending = 0; 2495 sk->sk_rcvlowat = 1; 2496 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 2497 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 2498 2499 sk->sk_stamp = ktime_set(-1L, 0); 2500 2501 #ifdef CONFIG_NET_RX_BUSY_POLL 2502 sk->sk_napi_id = 0; 2503 sk->sk_ll_usec = sysctl_net_busy_read; 2504 #endif 2505 2506 sk->sk_max_pacing_rate = ~0U; 2507 sk->sk_pacing_rate = ~0U; 2508 sk->sk_incoming_cpu = -1; 2509 /* 2510 * Before updating sk_refcnt, we must commit prior changes to memory 2511 * (Documentation/RCU/rculist_nulls.txt for details) 2512 */ 2513 smp_wmb(); 2514 atomic_set(&sk->sk_refcnt, 1); 2515 atomic_set(&sk->sk_drops, 0); 2516 } 2517 EXPORT_SYMBOL(sock_init_data); 2518 2519 void lock_sock_nested(struct sock *sk, int subclass) 2520 { 2521 might_sleep(); 2522 spin_lock_bh(&sk->sk_lock.slock); 2523 if (sk->sk_lock.owned) 2524 __lock_sock(sk); 2525 sk->sk_lock.owned = 1; 2526 spin_unlock(&sk->sk_lock.slock); 2527 /* 2528 * The sk_lock has mutex_lock() semantics here: 2529 */ 2530 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 2531 local_bh_enable(); 2532 } 2533 EXPORT_SYMBOL(lock_sock_nested); 2534 2535 void release_sock(struct sock *sk) 2536 { 2537 spin_lock_bh(&sk->sk_lock.slock); 2538 if (sk->sk_backlog.tail) 2539 __release_sock(sk); 2540 2541 /* Warning : release_cb() might need to release sk ownership, 2542 * ie call sock_release_ownership(sk) before us. 2543 */ 2544 if (sk->sk_prot->release_cb) 2545 sk->sk_prot->release_cb(sk); 2546 2547 sock_release_ownership(sk); 2548 if (waitqueue_active(&sk->sk_lock.wq)) 2549 wake_up(&sk->sk_lock.wq); 2550 spin_unlock_bh(&sk->sk_lock.slock); 2551 } 2552 EXPORT_SYMBOL(release_sock); 2553 2554 /** 2555 * lock_sock_fast - fast version of lock_sock 2556 * @sk: socket 2557 * 2558 * This version should be used for very small section, where process wont block 2559 * return false if fast path is taken 2560 * sk_lock.slock locked, owned = 0, BH disabled 2561 * return true if slow path is taken 2562 * sk_lock.slock unlocked, owned = 1, BH enabled 2563 */ 2564 bool lock_sock_fast(struct sock *sk) 2565 { 2566 might_sleep(); 2567 spin_lock_bh(&sk->sk_lock.slock); 2568 2569 if (!sk->sk_lock.owned) 2570 /* 2571 * Note : We must disable BH 2572 */ 2573 return false; 2574 2575 __lock_sock(sk); 2576 sk->sk_lock.owned = 1; 2577 spin_unlock(&sk->sk_lock.slock); 2578 /* 2579 * The sk_lock has mutex_lock() semantics here: 2580 */ 2581 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_); 2582 local_bh_enable(); 2583 return true; 2584 } 2585 EXPORT_SYMBOL(lock_sock_fast); 2586 2587 int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp) 2588 { 2589 struct timeval tv; 2590 if (!sock_flag(sk, SOCK_TIMESTAMP)) 2591 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2592 tv = ktime_to_timeval(sk->sk_stamp); 2593 if (tv.tv_sec == -1) 2594 return -ENOENT; 2595 if (tv.tv_sec == 0) { 2596 sk->sk_stamp = ktime_get_real(); 2597 tv = ktime_to_timeval(sk->sk_stamp); 2598 } 2599 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0; 2600 } 2601 EXPORT_SYMBOL(sock_get_timestamp); 2602 2603 int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp) 2604 { 2605 struct timespec ts; 2606 if (!sock_flag(sk, SOCK_TIMESTAMP)) 2607 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2608 ts = ktime_to_timespec(sk->sk_stamp); 2609 if (ts.tv_sec == -1) 2610 return -ENOENT; 2611 if (ts.tv_sec == 0) { 2612 sk->sk_stamp = ktime_get_real(); 2613 ts = ktime_to_timespec(sk->sk_stamp); 2614 } 2615 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0; 2616 } 2617 EXPORT_SYMBOL(sock_get_timestampns); 2618 2619 void sock_enable_timestamp(struct sock *sk, int flag) 2620 { 2621 if (!sock_flag(sk, flag)) { 2622 unsigned long previous_flags = sk->sk_flags; 2623 2624 sock_set_flag(sk, flag); 2625 /* 2626 * we just set one of the two flags which require net 2627 * time stamping, but time stamping might have been on 2628 * already because of the other one 2629 */ 2630 if (sock_needs_netstamp(sk) && 2631 !(previous_flags & SK_FLAGS_TIMESTAMP)) 2632 net_enable_timestamp(); 2633 } 2634 } 2635 2636 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 2637 int level, int type) 2638 { 2639 struct sock_exterr_skb *serr; 2640 struct sk_buff *skb; 2641 int copied, err; 2642 2643 err = -EAGAIN; 2644 skb = sock_dequeue_err_skb(sk); 2645 if (skb == NULL) 2646 goto out; 2647 2648 copied = skb->len; 2649 if (copied > len) { 2650 msg->msg_flags |= MSG_TRUNC; 2651 copied = len; 2652 } 2653 err = skb_copy_datagram_msg(skb, 0, msg, copied); 2654 if (err) 2655 goto out_free_skb; 2656 2657 sock_recv_timestamp(msg, sk, skb); 2658 2659 serr = SKB_EXT_ERR(skb); 2660 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 2661 2662 msg->msg_flags |= MSG_ERRQUEUE; 2663 err = copied; 2664 2665 out_free_skb: 2666 kfree_skb(skb); 2667 out: 2668 return err; 2669 } 2670 EXPORT_SYMBOL(sock_recv_errqueue); 2671 2672 /* 2673 * Get a socket option on an socket. 2674 * 2675 * FIX: POSIX 1003.1g is very ambiguous here. It states that 2676 * asynchronous errors should be reported by getsockopt. We assume 2677 * this means if you specify SO_ERROR (otherwise whats the point of it). 2678 */ 2679 int sock_common_getsockopt(struct socket *sock, int level, int optname, 2680 char __user *optval, int __user *optlen) 2681 { 2682 struct sock *sk = sock->sk; 2683 2684 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 2685 } 2686 EXPORT_SYMBOL(sock_common_getsockopt); 2687 2688 #ifdef CONFIG_COMPAT 2689 int compat_sock_common_getsockopt(struct socket *sock, int level, int optname, 2690 char __user *optval, int __user *optlen) 2691 { 2692 struct sock *sk = sock->sk; 2693 2694 if (sk->sk_prot->compat_getsockopt != NULL) 2695 return sk->sk_prot->compat_getsockopt(sk, level, optname, 2696 optval, optlen); 2697 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 2698 } 2699 EXPORT_SYMBOL(compat_sock_common_getsockopt); 2700 #endif 2701 2702 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 2703 int flags) 2704 { 2705 struct sock *sk = sock->sk; 2706 int addr_len = 0; 2707 int err; 2708 2709 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, 2710 flags & ~MSG_DONTWAIT, &addr_len); 2711 if (err >= 0) 2712 msg->msg_namelen = addr_len; 2713 return err; 2714 } 2715 EXPORT_SYMBOL(sock_common_recvmsg); 2716 2717 /* 2718 * Set socket options on an inet socket. 2719 */ 2720 int sock_common_setsockopt(struct socket *sock, int level, int optname, 2721 char __user *optval, unsigned int optlen) 2722 { 2723 struct sock *sk = sock->sk; 2724 2725 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 2726 } 2727 EXPORT_SYMBOL(sock_common_setsockopt); 2728 2729 #ifdef CONFIG_COMPAT 2730 int compat_sock_common_setsockopt(struct socket *sock, int level, int optname, 2731 char __user *optval, unsigned int optlen) 2732 { 2733 struct sock *sk = sock->sk; 2734 2735 if (sk->sk_prot->compat_setsockopt != NULL) 2736 return sk->sk_prot->compat_setsockopt(sk, level, optname, 2737 optval, optlen); 2738 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 2739 } 2740 EXPORT_SYMBOL(compat_sock_common_setsockopt); 2741 #endif 2742 2743 void sk_common_release(struct sock *sk) 2744 { 2745 if (sk->sk_prot->destroy) 2746 sk->sk_prot->destroy(sk); 2747 2748 /* 2749 * Observation: when sock_common_release is called, processes have 2750 * no access to socket. But net still has. 2751 * Step one, detach it from networking: 2752 * 2753 * A. Remove from hash tables. 2754 */ 2755 2756 sk->sk_prot->unhash(sk); 2757 2758 /* 2759 * In this point socket cannot receive new packets, but it is possible 2760 * that some packets are in flight because some CPU runs receiver and 2761 * did hash table lookup before we unhashed socket. They will achieve 2762 * receive queue and will be purged by socket destructor. 2763 * 2764 * Also we still have packets pending on receive queue and probably, 2765 * our own packets waiting in device queues. sock_destroy will drain 2766 * receive queue, but transmitted packets will delay socket destruction 2767 * until the last reference will be released. 2768 */ 2769 2770 sock_orphan(sk); 2771 2772 xfrm_sk_free_policy(sk); 2773 2774 sk_refcnt_debug_release(sk); 2775 2776 if (sk->sk_frag.page) { 2777 put_page(sk->sk_frag.page); 2778 sk->sk_frag.page = NULL; 2779 } 2780 2781 sock_put(sk); 2782 } 2783 EXPORT_SYMBOL(sk_common_release); 2784 2785 #ifdef CONFIG_PROC_FS 2786 #define PROTO_INUSE_NR 64 /* should be enough for the first time */ 2787 struct prot_inuse { 2788 int val[PROTO_INUSE_NR]; 2789 }; 2790 2791 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 2792 2793 #ifdef CONFIG_NET_NS 2794 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) 2795 { 2796 __this_cpu_add(net->core.inuse->val[prot->inuse_idx], val); 2797 } 2798 EXPORT_SYMBOL_GPL(sock_prot_inuse_add); 2799 2800 int sock_prot_inuse_get(struct net *net, struct proto *prot) 2801 { 2802 int cpu, idx = prot->inuse_idx; 2803 int res = 0; 2804 2805 for_each_possible_cpu(cpu) 2806 res += per_cpu_ptr(net->core.inuse, cpu)->val[idx]; 2807 2808 return res >= 0 ? res : 0; 2809 } 2810 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 2811 2812 static int __net_init sock_inuse_init_net(struct net *net) 2813 { 2814 net->core.inuse = alloc_percpu(struct prot_inuse); 2815 return net->core.inuse ? 0 : -ENOMEM; 2816 } 2817 2818 static void __net_exit sock_inuse_exit_net(struct net *net) 2819 { 2820 free_percpu(net->core.inuse); 2821 } 2822 2823 static struct pernet_operations net_inuse_ops = { 2824 .init = sock_inuse_init_net, 2825 .exit = sock_inuse_exit_net, 2826 }; 2827 2828 static __init int net_inuse_init(void) 2829 { 2830 if (register_pernet_subsys(&net_inuse_ops)) 2831 panic("Cannot initialize net inuse counters"); 2832 2833 return 0; 2834 } 2835 2836 core_initcall(net_inuse_init); 2837 #else 2838 static DEFINE_PER_CPU(struct prot_inuse, prot_inuse); 2839 2840 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) 2841 { 2842 __this_cpu_add(prot_inuse.val[prot->inuse_idx], val); 2843 } 2844 EXPORT_SYMBOL_GPL(sock_prot_inuse_add); 2845 2846 int sock_prot_inuse_get(struct net *net, struct proto *prot) 2847 { 2848 int cpu, idx = prot->inuse_idx; 2849 int res = 0; 2850 2851 for_each_possible_cpu(cpu) 2852 res += per_cpu(prot_inuse, cpu).val[idx]; 2853 2854 return res >= 0 ? res : 0; 2855 } 2856 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 2857 #endif 2858 2859 static void assign_proto_idx(struct proto *prot) 2860 { 2861 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 2862 2863 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 2864 pr_err("PROTO_INUSE_NR exhausted\n"); 2865 return; 2866 } 2867 2868 set_bit(prot->inuse_idx, proto_inuse_idx); 2869 } 2870 2871 static void release_proto_idx(struct proto *prot) 2872 { 2873 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 2874 clear_bit(prot->inuse_idx, proto_inuse_idx); 2875 } 2876 #else 2877 static inline void assign_proto_idx(struct proto *prot) 2878 { 2879 } 2880 2881 static inline void release_proto_idx(struct proto *prot) 2882 { 2883 } 2884 #endif 2885 2886 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 2887 { 2888 if (!rsk_prot) 2889 return; 2890 kfree(rsk_prot->slab_name); 2891 rsk_prot->slab_name = NULL; 2892 kmem_cache_destroy(rsk_prot->slab); 2893 rsk_prot->slab = NULL; 2894 } 2895 2896 static int req_prot_init(const struct proto *prot) 2897 { 2898 struct request_sock_ops *rsk_prot = prot->rsk_prot; 2899 2900 if (!rsk_prot) 2901 return 0; 2902 2903 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 2904 prot->name); 2905 if (!rsk_prot->slab_name) 2906 return -ENOMEM; 2907 2908 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 2909 rsk_prot->obj_size, 0, 2910 prot->slab_flags, NULL); 2911 2912 if (!rsk_prot->slab) { 2913 pr_crit("%s: Can't create request sock SLAB cache!\n", 2914 prot->name); 2915 return -ENOMEM; 2916 } 2917 return 0; 2918 } 2919 2920 int proto_register(struct proto *prot, int alloc_slab) 2921 { 2922 if (alloc_slab) { 2923 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0, 2924 SLAB_HWCACHE_ALIGN | prot->slab_flags, 2925 NULL); 2926 2927 if (prot->slab == NULL) { 2928 pr_crit("%s: Can't create sock SLAB cache!\n", 2929 prot->name); 2930 goto out; 2931 } 2932 2933 if (req_prot_init(prot)) 2934 goto out_free_request_sock_slab; 2935 2936 if (prot->twsk_prot != NULL) { 2937 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name); 2938 2939 if (prot->twsk_prot->twsk_slab_name == NULL) 2940 goto out_free_request_sock_slab; 2941 2942 prot->twsk_prot->twsk_slab = 2943 kmem_cache_create(prot->twsk_prot->twsk_slab_name, 2944 prot->twsk_prot->twsk_obj_size, 2945 0, 2946 prot->slab_flags, 2947 NULL); 2948 if (prot->twsk_prot->twsk_slab == NULL) 2949 goto out_free_timewait_sock_slab_name; 2950 } 2951 } 2952 2953 mutex_lock(&proto_list_mutex); 2954 list_add(&prot->node, &proto_list); 2955 assign_proto_idx(prot); 2956 mutex_unlock(&proto_list_mutex); 2957 return 0; 2958 2959 out_free_timewait_sock_slab_name: 2960 kfree(prot->twsk_prot->twsk_slab_name); 2961 out_free_request_sock_slab: 2962 req_prot_cleanup(prot->rsk_prot); 2963 2964 kmem_cache_destroy(prot->slab); 2965 prot->slab = NULL; 2966 out: 2967 return -ENOBUFS; 2968 } 2969 EXPORT_SYMBOL(proto_register); 2970 2971 void proto_unregister(struct proto *prot) 2972 { 2973 mutex_lock(&proto_list_mutex); 2974 release_proto_idx(prot); 2975 list_del(&prot->node); 2976 mutex_unlock(&proto_list_mutex); 2977 2978 kmem_cache_destroy(prot->slab); 2979 prot->slab = NULL; 2980 2981 req_prot_cleanup(prot->rsk_prot); 2982 2983 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) { 2984 kmem_cache_destroy(prot->twsk_prot->twsk_slab); 2985 kfree(prot->twsk_prot->twsk_slab_name); 2986 prot->twsk_prot->twsk_slab = NULL; 2987 } 2988 } 2989 EXPORT_SYMBOL(proto_unregister); 2990 2991 #ifdef CONFIG_PROC_FS 2992 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 2993 __acquires(proto_list_mutex) 2994 { 2995 mutex_lock(&proto_list_mutex); 2996 return seq_list_start_head(&proto_list, *pos); 2997 } 2998 2999 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3000 { 3001 return seq_list_next(v, &proto_list, pos); 3002 } 3003 3004 static void proto_seq_stop(struct seq_file *seq, void *v) 3005 __releases(proto_list_mutex) 3006 { 3007 mutex_unlock(&proto_list_mutex); 3008 } 3009 3010 static char proto_method_implemented(const void *method) 3011 { 3012 return method == NULL ? 'n' : 'y'; 3013 } 3014 static long sock_prot_memory_allocated(struct proto *proto) 3015 { 3016 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 3017 } 3018 3019 static char *sock_prot_memory_pressure(struct proto *proto) 3020 { 3021 return proto->memory_pressure != NULL ? 3022 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 3023 } 3024 3025 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 3026 { 3027 3028 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 3029 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 3030 proto->name, 3031 proto->obj_size, 3032 sock_prot_inuse_get(seq_file_net(seq), proto), 3033 sock_prot_memory_allocated(proto), 3034 sock_prot_memory_pressure(proto), 3035 proto->max_header, 3036 proto->slab == NULL ? "no" : "yes", 3037 module_name(proto->owner), 3038 proto_method_implemented(proto->close), 3039 proto_method_implemented(proto->connect), 3040 proto_method_implemented(proto->disconnect), 3041 proto_method_implemented(proto->accept), 3042 proto_method_implemented(proto->ioctl), 3043 proto_method_implemented(proto->init), 3044 proto_method_implemented(proto->destroy), 3045 proto_method_implemented(proto->shutdown), 3046 proto_method_implemented(proto->setsockopt), 3047 proto_method_implemented(proto->getsockopt), 3048 proto_method_implemented(proto->sendmsg), 3049 proto_method_implemented(proto->recvmsg), 3050 proto_method_implemented(proto->sendpage), 3051 proto_method_implemented(proto->bind), 3052 proto_method_implemented(proto->backlog_rcv), 3053 proto_method_implemented(proto->hash), 3054 proto_method_implemented(proto->unhash), 3055 proto_method_implemented(proto->get_port), 3056 proto_method_implemented(proto->enter_memory_pressure)); 3057 } 3058 3059 static int proto_seq_show(struct seq_file *seq, void *v) 3060 { 3061 if (v == &proto_list) 3062 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 3063 "protocol", 3064 "size", 3065 "sockets", 3066 "memory", 3067 "press", 3068 "maxhdr", 3069 "slab", 3070 "module", 3071 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n"); 3072 else 3073 proto_seq_printf(seq, list_entry(v, struct proto, node)); 3074 return 0; 3075 } 3076 3077 static const struct seq_operations proto_seq_ops = { 3078 .start = proto_seq_start, 3079 .next = proto_seq_next, 3080 .stop = proto_seq_stop, 3081 .show = proto_seq_show, 3082 }; 3083 3084 static int proto_seq_open(struct inode *inode, struct file *file) 3085 { 3086 return seq_open_net(inode, file, &proto_seq_ops, 3087 sizeof(struct seq_net_private)); 3088 } 3089 3090 static const struct file_operations proto_seq_fops = { 3091 .owner = THIS_MODULE, 3092 .open = proto_seq_open, 3093 .read = seq_read, 3094 .llseek = seq_lseek, 3095 .release = seq_release_net, 3096 }; 3097 3098 static __net_init int proto_init_net(struct net *net) 3099 { 3100 if (!proc_create("protocols", S_IRUGO, net->proc_net, &proto_seq_fops)) 3101 return -ENOMEM; 3102 3103 return 0; 3104 } 3105 3106 static __net_exit void proto_exit_net(struct net *net) 3107 { 3108 remove_proc_entry("protocols", net->proc_net); 3109 } 3110 3111 3112 static __net_initdata struct pernet_operations proto_net_ops = { 3113 .init = proto_init_net, 3114 .exit = proto_exit_net, 3115 }; 3116 3117 static int __init proto_init(void) 3118 { 3119 return register_pernet_subsys(&proto_net_ops); 3120 } 3121 3122 subsys_initcall(proto_init); 3123 3124 #endif /* PROC_FS */ 3125