1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Generic socket support routines. Memory allocators, socket lock/release 7 * handler for protocols to use and generic option handler. 8 * 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 * 85 * 86 * This program is free software; you can redistribute it and/or 87 * modify it under the terms of the GNU General Public License 88 * as published by the Free Software Foundation; either version 89 * 2 of the License, or (at your option) any later version. 90 */ 91 92 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 93 94 #include <asm/unaligned.h> 95 #include <linux/capability.h> 96 #include <linux/errno.h> 97 #include <linux/errqueue.h> 98 #include <linux/types.h> 99 #include <linux/socket.h> 100 #include <linux/in.h> 101 #include <linux/kernel.h> 102 #include <linux/module.h> 103 #include <linux/proc_fs.h> 104 #include <linux/seq_file.h> 105 #include <linux/sched.h> 106 #include <linux/sched/mm.h> 107 #include <linux/timer.h> 108 #include <linux/string.h> 109 #include <linux/sockios.h> 110 #include <linux/net.h> 111 #include <linux/mm.h> 112 #include <linux/slab.h> 113 #include <linux/interrupt.h> 114 #include <linux/poll.h> 115 #include <linux/tcp.h> 116 #include <linux/init.h> 117 #include <linux/highmem.h> 118 #include <linux/user_namespace.h> 119 #include <linux/static_key.h> 120 #include <linux/memcontrol.h> 121 #include <linux/prefetch.h> 122 123 #include <linux/uaccess.h> 124 125 #include <linux/netdevice.h> 126 #include <net/protocol.h> 127 #include <linux/skbuff.h> 128 #include <net/net_namespace.h> 129 #include <net/request_sock.h> 130 #include <net/sock.h> 131 #include <linux/net_tstamp.h> 132 #include <net/xfrm.h> 133 #include <linux/ipsec.h> 134 #include <net/cls_cgroup.h> 135 #include <net/netprio_cgroup.h> 136 #include <linux/sock_diag.h> 137 138 #include <linux/filter.h> 139 #include <net/sock_reuseport.h> 140 141 #include <trace/events/sock.h> 142 143 #include <net/tcp.h> 144 #include <net/busy_poll.h> 145 146 static DEFINE_MUTEX(proto_list_mutex); 147 static LIST_HEAD(proto_list); 148 149 static void sock_inuse_add(struct net *net, int val); 150 151 /** 152 * sk_ns_capable - General socket capability test 153 * @sk: Socket to use a capability on or through 154 * @user_ns: The user namespace of the capability to use 155 * @cap: The capability to use 156 * 157 * Test to see if the opener of the socket had when the socket was 158 * created and the current process has the capability @cap in the user 159 * namespace @user_ns. 160 */ 161 bool sk_ns_capable(const struct sock *sk, 162 struct user_namespace *user_ns, int cap) 163 { 164 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 165 ns_capable(user_ns, cap); 166 } 167 EXPORT_SYMBOL(sk_ns_capable); 168 169 /** 170 * sk_capable - Socket global capability test 171 * @sk: Socket to use a capability on or through 172 * @cap: The global capability to use 173 * 174 * Test to see if the opener of the socket had when the socket was 175 * created and the current process has the capability @cap in all user 176 * namespaces. 177 */ 178 bool sk_capable(const struct sock *sk, int cap) 179 { 180 return sk_ns_capable(sk, &init_user_ns, cap); 181 } 182 EXPORT_SYMBOL(sk_capable); 183 184 /** 185 * sk_net_capable - Network namespace socket capability test 186 * @sk: Socket to use a capability on or through 187 * @cap: The capability to use 188 * 189 * Test to see if the opener of the socket had when the socket was created 190 * and the current process has the capability @cap over the network namespace 191 * the socket is a member of. 192 */ 193 bool sk_net_capable(const struct sock *sk, int cap) 194 { 195 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 196 } 197 EXPORT_SYMBOL(sk_net_capable); 198 199 /* 200 * Each address family might have different locking rules, so we have 201 * one slock key per address family and separate keys for internal and 202 * userspace sockets. 203 */ 204 static struct lock_class_key af_family_keys[AF_MAX]; 205 static struct lock_class_key af_family_kern_keys[AF_MAX]; 206 static struct lock_class_key af_family_slock_keys[AF_MAX]; 207 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 208 209 /* 210 * Make lock validator output more readable. (we pre-construct these 211 * strings build-time, so that runtime initialization of socket 212 * locks is fast): 213 */ 214 215 #define _sock_locks(x) \ 216 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 217 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 218 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 219 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 220 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 221 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 222 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 223 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 224 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 225 x "27" , x "28" , x "AF_CAN" , \ 226 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 227 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 228 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 229 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 230 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 231 x "AF_MAX" 232 233 static const char *const af_family_key_strings[AF_MAX+1] = { 234 _sock_locks("sk_lock-") 235 }; 236 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 237 _sock_locks("slock-") 238 }; 239 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 240 _sock_locks("clock-") 241 }; 242 243 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 244 _sock_locks("k-sk_lock-") 245 }; 246 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 247 _sock_locks("k-slock-") 248 }; 249 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 250 _sock_locks("k-clock-") 251 }; 252 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 253 _sock_locks("rlock-") 254 }; 255 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 256 _sock_locks("wlock-") 257 }; 258 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 259 _sock_locks("elock-") 260 }; 261 262 /* 263 * sk_callback_lock and sk queues locking rules are per-address-family, 264 * so split the lock classes by using a per-AF key: 265 */ 266 static struct lock_class_key af_callback_keys[AF_MAX]; 267 static struct lock_class_key af_rlock_keys[AF_MAX]; 268 static struct lock_class_key af_wlock_keys[AF_MAX]; 269 static struct lock_class_key af_elock_keys[AF_MAX]; 270 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 271 272 /* Run time adjustable parameters. */ 273 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 274 EXPORT_SYMBOL(sysctl_wmem_max); 275 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 276 EXPORT_SYMBOL(sysctl_rmem_max); 277 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 278 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 279 280 /* Maximal space eaten by iovec or ancillary data plus some space */ 281 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 282 EXPORT_SYMBOL(sysctl_optmem_max); 283 284 int sysctl_tstamp_allow_data __read_mostly = 1; 285 286 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 287 EXPORT_SYMBOL_GPL(memalloc_socks_key); 288 289 /** 290 * sk_set_memalloc - sets %SOCK_MEMALLOC 291 * @sk: socket to set it on 292 * 293 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 294 * It's the responsibility of the admin to adjust min_free_kbytes 295 * to meet the requirements 296 */ 297 void sk_set_memalloc(struct sock *sk) 298 { 299 sock_set_flag(sk, SOCK_MEMALLOC); 300 sk->sk_allocation |= __GFP_MEMALLOC; 301 static_branch_inc(&memalloc_socks_key); 302 } 303 EXPORT_SYMBOL_GPL(sk_set_memalloc); 304 305 void sk_clear_memalloc(struct sock *sk) 306 { 307 sock_reset_flag(sk, SOCK_MEMALLOC); 308 sk->sk_allocation &= ~__GFP_MEMALLOC; 309 static_branch_dec(&memalloc_socks_key); 310 311 /* 312 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 313 * progress of swapping. SOCK_MEMALLOC may be cleared while 314 * it has rmem allocations due to the last swapfile being deactivated 315 * but there is a risk that the socket is unusable due to exceeding 316 * the rmem limits. Reclaim the reserves and obey rmem limits again. 317 */ 318 sk_mem_reclaim(sk); 319 } 320 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 321 322 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 323 { 324 int ret; 325 unsigned int noreclaim_flag; 326 327 /* these should have been dropped before queueing */ 328 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 329 330 noreclaim_flag = memalloc_noreclaim_save(); 331 ret = sk->sk_backlog_rcv(sk, skb); 332 memalloc_noreclaim_restore(noreclaim_flag); 333 334 return ret; 335 } 336 EXPORT_SYMBOL(__sk_backlog_rcv); 337 338 static int sock_get_timeout(long timeo, void *optval, bool old_timeval) 339 { 340 struct __kernel_sock_timeval tv; 341 int size; 342 343 if (timeo == MAX_SCHEDULE_TIMEOUT) { 344 tv.tv_sec = 0; 345 tv.tv_usec = 0; 346 } else { 347 tv.tv_sec = timeo / HZ; 348 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 349 } 350 351 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 352 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 353 *(struct old_timeval32 *)optval = tv32; 354 return sizeof(tv32); 355 } 356 357 if (old_timeval) { 358 struct __kernel_old_timeval old_tv; 359 old_tv.tv_sec = tv.tv_sec; 360 old_tv.tv_usec = tv.tv_usec; 361 *(struct __kernel_old_timeval *)optval = old_tv; 362 size = sizeof(old_tv); 363 } else { 364 *(struct __kernel_sock_timeval *)optval = tv; 365 size = sizeof(tv); 366 } 367 368 return size; 369 } 370 371 static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen, bool old_timeval) 372 { 373 struct __kernel_sock_timeval tv; 374 375 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 376 struct old_timeval32 tv32; 377 378 if (optlen < sizeof(tv32)) 379 return -EINVAL; 380 381 if (copy_from_user(&tv32, optval, sizeof(tv32))) 382 return -EFAULT; 383 tv.tv_sec = tv32.tv_sec; 384 tv.tv_usec = tv32.tv_usec; 385 } else if (old_timeval) { 386 struct __kernel_old_timeval old_tv; 387 388 if (optlen < sizeof(old_tv)) 389 return -EINVAL; 390 if (copy_from_user(&old_tv, optval, sizeof(old_tv))) 391 return -EFAULT; 392 tv.tv_sec = old_tv.tv_sec; 393 tv.tv_usec = old_tv.tv_usec; 394 } else { 395 if (optlen < sizeof(tv)) 396 return -EINVAL; 397 if (copy_from_user(&tv, optval, sizeof(tv))) 398 return -EFAULT; 399 } 400 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 401 return -EDOM; 402 403 if (tv.tv_sec < 0) { 404 static int warned __read_mostly; 405 406 *timeo_p = 0; 407 if (warned < 10 && net_ratelimit()) { 408 warned++; 409 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 410 __func__, current->comm, task_pid_nr(current)); 411 } 412 return 0; 413 } 414 *timeo_p = MAX_SCHEDULE_TIMEOUT; 415 if (tv.tv_sec == 0 && tv.tv_usec == 0) 416 return 0; 417 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1)) 418 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, USEC_PER_SEC / HZ); 419 return 0; 420 } 421 422 static void sock_warn_obsolete_bsdism(const char *name) 423 { 424 static int warned; 425 static char warncomm[TASK_COMM_LEN]; 426 if (strcmp(warncomm, current->comm) && warned < 5) { 427 strcpy(warncomm, current->comm); 428 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n", 429 warncomm, name); 430 warned++; 431 } 432 } 433 434 static bool sock_needs_netstamp(const struct sock *sk) 435 { 436 switch (sk->sk_family) { 437 case AF_UNSPEC: 438 case AF_UNIX: 439 return false; 440 default: 441 return true; 442 } 443 } 444 445 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 446 { 447 if (sk->sk_flags & flags) { 448 sk->sk_flags &= ~flags; 449 if (sock_needs_netstamp(sk) && 450 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 451 net_disable_timestamp(); 452 } 453 } 454 455 456 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 457 { 458 unsigned long flags; 459 struct sk_buff_head *list = &sk->sk_receive_queue; 460 461 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { 462 atomic_inc(&sk->sk_drops); 463 trace_sock_rcvqueue_full(sk, skb); 464 return -ENOMEM; 465 } 466 467 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 468 atomic_inc(&sk->sk_drops); 469 return -ENOBUFS; 470 } 471 472 skb->dev = NULL; 473 skb_set_owner_r(skb, sk); 474 475 /* we escape from rcu protected region, make sure we dont leak 476 * a norefcounted dst 477 */ 478 skb_dst_force(skb); 479 480 spin_lock_irqsave(&list->lock, flags); 481 sock_skb_set_dropcount(sk, skb); 482 __skb_queue_tail(list, skb); 483 spin_unlock_irqrestore(&list->lock, flags); 484 485 if (!sock_flag(sk, SOCK_DEAD)) 486 sk->sk_data_ready(sk); 487 return 0; 488 } 489 EXPORT_SYMBOL(__sock_queue_rcv_skb); 490 491 int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 492 { 493 int err; 494 495 err = sk_filter(sk, skb); 496 if (err) 497 return err; 498 499 return __sock_queue_rcv_skb(sk, skb); 500 } 501 EXPORT_SYMBOL(sock_queue_rcv_skb); 502 503 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 504 const int nested, unsigned int trim_cap, bool refcounted) 505 { 506 int rc = NET_RX_SUCCESS; 507 508 if (sk_filter_trim_cap(sk, skb, trim_cap)) 509 goto discard_and_relse; 510 511 skb->dev = NULL; 512 513 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) { 514 atomic_inc(&sk->sk_drops); 515 goto discard_and_relse; 516 } 517 if (nested) 518 bh_lock_sock_nested(sk); 519 else 520 bh_lock_sock(sk); 521 if (!sock_owned_by_user(sk)) { 522 /* 523 * trylock + unlock semantics: 524 */ 525 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 526 527 rc = sk_backlog_rcv(sk, skb); 528 529 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_); 530 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) { 531 bh_unlock_sock(sk); 532 atomic_inc(&sk->sk_drops); 533 goto discard_and_relse; 534 } 535 536 bh_unlock_sock(sk); 537 out: 538 if (refcounted) 539 sock_put(sk); 540 return rc; 541 discard_and_relse: 542 kfree_skb(skb); 543 goto out; 544 } 545 EXPORT_SYMBOL(__sk_receive_skb); 546 547 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 548 { 549 struct dst_entry *dst = __sk_dst_get(sk); 550 551 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 552 sk_tx_queue_clear(sk); 553 sk->sk_dst_pending_confirm = 0; 554 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 555 dst_release(dst); 556 return NULL; 557 } 558 559 return dst; 560 } 561 EXPORT_SYMBOL(__sk_dst_check); 562 563 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 564 { 565 struct dst_entry *dst = sk_dst_get(sk); 566 567 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) { 568 sk_dst_reset(sk); 569 dst_release(dst); 570 return NULL; 571 } 572 573 return dst; 574 } 575 EXPORT_SYMBOL(sk_dst_check); 576 577 static int sock_setbindtodevice_locked(struct sock *sk, int ifindex) 578 { 579 int ret = -ENOPROTOOPT; 580 #ifdef CONFIG_NETDEVICES 581 struct net *net = sock_net(sk); 582 583 /* Sorry... */ 584 ret = -EPERM; 585 if (!ns_capable(net->user_ns, CAP_NET_RAW)) 586 goto out; 587 588 ret = -EINVAL; 589 if (ifindex < 0) 590 goto out; 591 592 sk->sk_bound_dev_if = ifindex; 593 if (sk->sk_prot->rehash) 594 sk->sk_prot->rehash(sk); 595 sk_dst_reset(sk); 596 597 ret = 0; 598 599 out: 600 #endif 601 602 return ret; 603 } 604 605 static int sock_setbindtodevice(struct sock *sk, char __user *optval, 606 int optlen) 607 { 608 int ret = -ENOPROTOOPT; 609 #ifdef CONFIG_NETDEVICES 610 struct net *net = sock_net(sk); 611 char devname[IFNAMSIZ]; 612 int index; 613 614 ret = -EINVAL; 615 if (optlen < 0) 616 goto out; 617 618 /* Bind this socket to a particular device like "eth0", 619 * as specified in the passed interface name. If the 620 * name is "" or the option length is zero the socket 621 * is not bound. 622 */ 623 if (optlen > IFNAMSIZ - 1) 624 optlen = IFNAMSIZ - 1; 625 memset(devname, 0, sizeof(devname)); 626 627 ret = -EFAULT; 628 if (copy_from_user(devname, optval, optlen)) 629 goto out; 630 631 index = 0; 632 if (devname[0] != '\0') { 633 struct net_device *dev; 634 635 rcu_read_lock(); 636 dev = dev_get_by_name_rcu(net, devname); 637 if (dev) 638 index = dev->ifindex; 639 rcu_read_unlock(); 640 ret = -ENODEV; 641 if (!dev) 642 goto out; 643 } 644 645 lock_sock(sk); 646 ret = sock_setbindtodevice_locked(sk, index); 647 release_sock(sk); 648 649 out: 650 #endif 651 652 return ret; 653 } 654 655 static int sock_getbindtodevice(struct sock *sk, char __user *optval, 656 int __user *optlen, int len) 657 { 658 int ret = -ENOPROTOOPT; 659 #ifdef CONFIG_NETDEVICES 660 struct net *net = sock_net(sk); 661 char devname[IFNAMSIZ]; 662 663 if (sk->sk_bound_dev_if == 0) { 664 len = 0; 665 goto zero; 666 } 667 668 ret = -EINVAL; 669 if (len < IFNAMSIZ) 670 goto out; 671 672 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if); 673 if (ret) 674 goto out; 675 676 len = strlen(devname) + 1; 677 678 ret = -EFAULT; 679 if (copy_to_user(optval, devname, len)) 680 goto out; 681 682 zero: 683 ret = -EFAULT; 684 if (put_user(len, optlen)) 685 goto out; 686 687 ret = 0; 688 689 out: 690 #endif 691 692 return ret; 693 } 694 695 static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool) 696 { 697 if (valbool) 698 sock_set_flag(sk, bit); 699 else 700 sock_reset_flag(sk, bit); 701 } 702 703 bool sk_mc_loop(struct sock *sk) 704 { 705 if (dev_recursion_level()) 706 return false; 707 if (!sk) 708 return true; 709 switch (sk->sk_family) { 710 case AF_INET: 711 return inet_sk(sk)->mc_loop; 712 #if IS_ENABLED(CONFIG_IPV6) 713 case AF_INET6: 714 return inet6_sk(sk)->mc_loop; 715 #endif 716 } 717 WARN_ON(1); 718 return true; 719 } 720 EXPORT_SYMBOL(sk_mc_loop); 721 722 /* 723 * This is meant for all protocols to use and covers goings on 724 * at the socket level. Everything here is generic. 725 */ 726 727 int sock_setsockopt(struct socket *sock, int level, int optname, 728 char __user *optval, unsigned int optlen) 729 { 730 struct sock_txtime sk_txtime; 731 struct sock *sk = sock->sk; 732 int val; 733 int valbool; 734 struct linger ling; 735 int ret = 0; 736 737 /* 738 * Options without arguments 739 */ 740 741 if (optname == SO_BINDTODEVICE) 742 return sock_setbindtodevice(sk, optval, optlen); 743 744 if (optlen < sizeof(int)) 745 return -EINVAL; 746 747 if (get_user(val, (int __user *)optval)) 748 return -EFAULT; 749 750 valbool = val ? 1 : 0; 751 752 lock_sock(sk); 753 754 switch (optname) { 755 case SO_DEBUG: 756 if (val && !capable(CAP_NET_ADMIN)) 757 ret = -EACCES; 758 else 759 sock_valbool_flag(sk, SOCK_DBG, valbool); 760 break; 761 case SO_REUSEADDR: 762 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 763 break; 764 case SO_REUSEPORT: 765 sk->sk_reuseport = valbool; 766 break; 767 case SO_TYPE: 768 case SO_PROTOCOL: 769 case SO_DOMAIN: 770 case SO_ERROR: 771 ret = -ENOPROTOOPT; 772 break; 773 case SO_DONTROUTE: 774 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 775 sk_dst_reset(sk); 776 break; 777 case SO_BROADCAST: 778 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 779 break; 780 case SO_SNDBUF: 781 /* Don't error on this BSD doesn't and if you think 782 * about it this is right. Otherwise apps have to 783 * play 'guess the biggest size' games. RCVBUF/SNDBUF 784 * are treated in BSD as hints 785 */ 786 val = min_t(u32, val, sysctl_wmem_max); 787 set_sndbuf: 788 /* Ensure val * 2 fits into an int, to prevent max_t() 789 * from treating it as a negative value. 790 */ 791 val = min_t(int, val, INT_MAX / 2); 792 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 793 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); 794 /* Wake up sending tasks if we upped the value. */ 795 sk->sk_write_space(sk); 796 break; 797 798 case SO_SNDBUFFORCE: 799 if (!capable(CAP_NET_ADMIN)) { 800 ret = -EPERM; 801 break; 802 } 803 804 /* No negative values (to prevent underflow, as val will be 805 * multiplied by 2). 806 */ 807 if (val < 0) 808 val = 0; 809 goto set_sndbuf; 810 811 case SO_RCVBUF: 812 /* Don't error on this BSD doesn't and if you think 813 * about it this is right. Otherwise apps have to 814 * play 'guess the biggest size' games. RCVBUF/SNDBUF 815 * are treated in BSD as hints 816 */ 817 val = min_t(u32, val, sysctl_rmem_max); 818 set_rcvbuf: 819 /* Ensure val * 2 fits into an int, to prevent max_t() 820 * from treating it as a negative value. 821 */ 822 val = min_t(int, val, INT_MAX / 2); 823 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 824 /* 825 * We double it on the way in to account for 826 * "struct sk_buff" etc. overhead. Applications 827 * assume that the SO_RCVBUF setting they make will 828 * allow that much actual data to be received on that 829 * socket. 830 * 831 * Applications are unaware that "struct sk_buff" and 832 * other overheads allocate from the receive buffer 833 * during socket buffer allocation. 834 * 835 * And after considering the possible alternatives, 836 * returning the value we actually used in getsockopt 837 * is the most desirable behavior. 838 */ 839 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF); 840 break; 841 842 case SO_RCVBUFFORCE: 843 if (!capable(CAP_NET_ADMIN)) { 844 ret = -EPERM; 845 break; 846 } 847 848 /* No negative values (to prevent underflow, as val will be 849 * multiplied by 2). 850 */ 851 if (val < 0) 852 val = 0; 853 goto set_rcvbuf; 854 855 case SO_KEEPALIVE: 856 if (sk->sk_prot->keepalive) 857 sk->sk_prot->keepalive(sk, valbool); 858 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 859 break; 860 861 case SO_OOBINLINE: 862 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 863 break; 864 865 case SO_NO_CHECK: 866 sk->sk_no_check_tx = valbool; 867 break; 868 869 case SO_PRIORITY: 870 if ((val >= 0 && val <= 6) || 871 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 872 sk->sk_priority = val; 873 else 874 ret = -EPERM; 875 break; 876 877 case SO_LINGER: 878 if (optlen < sizeof(ling)) { 879 ret = -EINVAL; /* 1003.1g */ 880 break; 881 } 882 if (copy_from_user(&ling, optval, sizeof(ling))) { 883 ret = -EFAULT; 884 break; 885 } 886 if (!ling.l_onoff) 887 sock_reset_flag(sk, SOCK_LINGER); 888 else { 889 #if (BITS_PER_LONG == 32) 890 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ) 891 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT; 892 else 893 #endif 894 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ; 895 sock_set_flag(sk, SOCK_LINGER); 896 } 897 break; 898 899 case SO_BSDCOMPAT: 900 sock_warn_obsolete_bsdism("setsockopt"); 901 break; 902 903 case SO_PASSCRED: 904 if (valbool) 905 set_bit(SOCK_PASSCRED, &sock->flags); 906 else 907 clear_bit(SOCK_PASSCRED, &sock->flags); 908 break; 909 910 case SO_TIMESTAMP_OLD: 911 case SO_TIMESTAMP_NEW: 912 case SO_TIMESTAMPNS_OLD: 913 case SO_TIMESTAMPNS_NEW: 914 if (valbool) { 915 if (optname == SO_TIMESTAMP_NEW || optname == SO_TIMESTAMPNS_NEW) 916 sock_set_flag(sk, SOCK_TSTAMP_NEW); 917 else 918 sock_reset_flag(sk, SOCK_TSTAMP_NEW); 919 920 if (optname == SO_TIMESTAMP_OLD || optname == SO_TIMESTAMP_NEW) 921 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 922 else 923 sock_set_flag(sk, SOCK_RCVTSTAMPNS); 924 sock_set_flag(sk, SOCK_RCVTSTAMP); 925 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 926 } else { 927 sock_reset_flag(sk, SOCK_RCVTSTAMP); 928 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 929 sock_reset_flag(sk, SOCK_TSTAMP_NEW); 930 } 931 break; 932 933 case SO_TIMESTAMPING_NEW: 934 sock_set_flag(sk, SOCK_TSTAMP_NEW); 935 /* fall through */ 936 case SO_TIMESTAMPING_OLD: 937 if (val & ~SOF_TIMESTAMPING_MASK) { 938 ret = -EINVAL; 939 break; 940 } 941 942 if (val & SOF_TIMESTAMPING_OPT_ID && 943 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 944 if (sk->sk_protocol == IPPROTO_TCP && 945 sk->sk_type == SOCK_STREAM) { 946 if ((1 << sk->sk_state) & 947 (TCPF_CLOSE | TCPF_LISTEN)) { 948 ret = -EINVAL; 949 break; 950 } 951 sk->sk_tskey = tcp_sk(sk)->snd_una; 952 } else { 953 sk->sk_tskey = 0; 954 } 955 } 956 957 if (val & SOF_TIMESTAMPING_OPT_STATS && 958 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) { 959 ret = -EINVAL; 960 break; 961 } 962 963 sk->sk_tsflags = val; 964 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 965 sock_enable_timestamp(sk, 966 SOCK_TIMESTAMPING_RX_SOFTWARE); 967 else { 968 if (optname == SO_TIMESTAMPING_NEW) 969 sock_reset_flag(sk, SOCK_TSTAMP_NEW); 970 971 sock_disable_timestamp(sk, 972 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 973 } 974 break; 975 976 case SO_RCVLOWAT: 977 if (val < 0) 978 val = INT_MAX; 979 if (sock->ops->set_rcvlowat) 980 ret = sock->ops->set_rcvlowat(sk, val); 981 else 982 sk->sk_rcvlowat = val ? : 1; 983 break; 984 985 case SO_RCVTIMEO_OLD: 986 case SO_RCVTIMEO_NEW: 987 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen, optname == SO_RCVTIMEO_OLD); 988 break; 989 990 case SO_SNDTIMEO_OLD: 991 case SO_SNDTIMEO_NEW: 992 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen, optname == SO_SNDTIMEO_OLD); 993 break; 994 995 case SO_ATTACH_FILTER: 996 ret = -EINVAL; 997 if (optlen == sizeof(struct sock_fprog)) { 998 struct sock_fprog fprog; 999 1000 ret = -EFAULT; 1001 if (copy_from_user(&fprog, optval, sizeof(fprog))) 1002 break; 1003 1004 ret = sk_attach_filter(&fprog, sk); 1005 } 1006 break; 1007 1008 case SO_ATTACH_BPF: 1009 ret = -EINVAL; 1010 if (optlen == sizeof(u32)) { 1011 u32 ufd; 1012 1013 ret = -EFAULT; 1014 if (copy_from_user(&ufd, optval, sizeof(ufd))) 1015 break; 1016 1017 ret = sk_attach_bpf(ufd, sk); 1018 } 1019 break; 1020 1021 case SO_ATTACH_REUSEPORT_CBPF: 1022 ret = -EINVAL; 1023 if (optlen == sizeof(struct sock_fprog)) { 1024 struct sock_fprog fprog; 1025 1026 ret = -EFAULT; 1027 if (copy_from_user(&fprog, optval, sizeof(fprog))) 1028 break; 1029 1030 ret = sk_reuseport_attach_filter(&fprog, sk); 1031 } 1032 break; 1033 1034 case SO_ATTACH_REUSEPORT_EBPF: 1035 ret = -EINVAL; 1036 if (optlen == sizeof(u32)) { 1037 u32 ufd; 1038 1039 ret = -EFAULT; 1040 if (copy_from_user(&ufd, optval, sizeof(ufd))) 1041 break; 1042 1043 ret = sk_reuseport_attach_bpf(ufd, sk); 1044 } 1045 break; 1046 1047 case SO_DETACH_FILTER: 1048 ret = sk_detach_filter(sk); 1049 break; 1050 1051 case SO_LOCK_FILTER: 1052 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1053 ret = -EPERM; 1054 else 1055 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1056 break; 1057 1058 case SO_PASSSEC: 1059 if (valbool) 1060 set_bit(SOCK_PASSSEC, &sock->flags); 1061 else 1062 clear_bit(SOCK_PASSSEC, &sock->flags); 1063 break; 1064 case SO_MARK: 1065 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1066 ret = -EPERM; 1067 } else if (val != sk->sk_mark) { 1068 sk->sk_mark = val; 1069 sk_dst_reset(sk); 1070 } 1071 break; 1072 1073 case SO_RXQ_OVFL: 1074 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1075 break; 1076 1077 case SO_WIFI_STATUS: 1078 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1079 break; 1080 1081 case SO_PEEK_OFF: 1082 if (sock->ops->set_peek_off) 1083 ret = sock->ops->set_peek_off(sk, val); 1084 else 1085 ret = -EOPNOTSUPP; 1086 break; 1087 1088 case SO_NOFCS: 1089 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1090 break; 1091 1092 case SO_SELECT_ERR_QUEUE: 1093 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1094 break; 1095 1096 #ifdef CONFIG_NET_RX_BUSY_POLL 1097 case SO_BUSY_POLL: 1098 /* allow unprivileged users to decrease the value */ 1099 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN)) 1100 ret = -EPERM; 1101 else { 1102 if (val < 0) 1103 ret = -EINVAL; 1104 else 1105 sk->sk_ll_usec = val; 1106 } 1107 break; 1108 #endif 1109 1110 case SO_MAX_PACING_RATE: 1111 { 1112 unsigned long ulval = (val == ~0U) ? ~0UL : val; 1113 1114 if (sizeof(ulval) != sizeof(val) && 1115 optlen >= sizeof(ulval) && 1116 get_user(ulval, (unsigned long __user *)optval)) { 1117 ret = -EFAULT; 1118 break; 1119 } 1120 if (ulval != ~0UL) 1121 cmpxchg(&sk->sk_pacing_status, 1122 SK_PACING_NONE, 1123 SK_PACING_NEEDED); 1124 sk->sk_max_pacing_rate = ulval; 1125 sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval); 1126 break; 1127 } 1128 case SO_INCOMING_CPU: 1129 sk->sk_incoming_cpu = val; 1130 break; 1131 1132 case SO_CNX_ADVICE: 1133 if (val == 1) 1134 dst_negative_advice(sk); 1135 break; 1136 1137 case SO_ZEROCOPY: 1138 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1139 if (!((sk->sk_type == SOCK_STREAM && 1140 sk->sk_protocol == IPPROTO_TCP) || 1141 (sk->sk_type == SOCK_DGRAM && 1142 sk->sk_protocol == IPPROTO_UDP))) 1143 ret = -ENOTSUPP; 1144 } else if (sk->sk_family != PF_RDS) { 1145 ret = -ENOTSUPP; 1146 } 1147 if (!ret) { 1148 if (val < 0 || val > 1) 1149 ret = -EINVAL; 1150 else 1151 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1152 } 1153 break; 1154 1155 case SO_TXTIME: 1156 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1157 ret = -EPERM; 1158 } else if (optlen != sizeof(struct sock_txtime)) { 1159 ret = -EINVAL; 1160 } else if (copy_from_user(&sk_txtime, optval, 1161 sizeof(struct sock_txtime))) { 1162 ret = -EFAULT; 1163 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1164 ret = -EINVAL; 1165 } else { 1166 sock_valbool_flag(sk, SOCK_TXTIME, true); 1167 sk->sk_clockid = sk_txtime.clockid; 1168 sk->sk_txtime_deadline_mode = 1169 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1170 sk->sk_txtime_report_errors = 1171 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1172 } 1173 break; 1174 1175 case SO_BINDTOIFINDEX: 1176 ret = sock_setbindtodevice_locked(sk, val); 1177 break; 1178 1179 default: 1180 ret = -ENOPROTOOPT; 1181 break; 1182 } 1183 release_sock(sk); 1184 return ret; 1185 } 1186 EXPORT_SYMBOL(sock_setsockopt); 1187 1188 1189 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1190 struct ucred *ucred) 1191 { 1192 ucred->pid = pid_vnr(pid); 1193 ucred->uid = ucred->gid = -1; 1194 if (cred) { 1195 struct user_namespace *current_ns = current_user_ns(); 1196 1197 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1198 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1199 } 1200 } 1201 1202 static int groups_to_user(gid_t __user *dst, const struct group_info *src) 1203 { 1204 struct user_namespace *user_ns = current_user_ns(); 1205 int i; 1206 1207 for (i = 0; i < src->ngroups; i++) 1208 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i)) 1209 return -EFAULT; 1210 1211 return 0; 1212 } 1213 1214 int sock_getsockopt(struct socket *sock, int level, int optname, 1215 char __user *optval, int __user *optlen) 1216 { 1217 struct sock *sk = sock->sk; 1218 1219 union { 1220 int val; 1221 u64 val64; 1222 unsigned long ulval; 1223 struct linger ling; 1224 struct old_timeval32 tm32; 1225 struct __kernel_old_timeval tm; 1226 struct __kernel_sock_timeval stm; 1227 struct sock_txtime txtime; 1228 } v; 1229 1230 int lv = sizeof(int); 1231 int len; 1232 1233 if (get_user(len, optlen)) 1234 return -EFAULT; 1235 if (len < 0) 1236 return -EINVAL; 1237 1238 memset(&v, 0, sizeof(v)); 1239 1240 switch (optname) { 1241 case SO_DEBUG: 1242 v.val = sock_flag(sk, SOCK_DBG); 1243 break; 1244 1245 case SO_DONTROUTE: 1246 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1247 break; 1248 1249 case SO_BROADCAST: 1250 v.val = sock_flag(sk, SOCK_BROADCAST); 1251 break; 1252 1253 case SO_SNDBUF: 1254 v.val = sk->sk_sndbuf; 1255 break; 1256 1257 case SO_RCVBUF: 1258 v.val = sk->sk_rcvbuf; 1259 break; 1260 1261 case SO_REUSEADDR: 1262 v.val = sk->sk_reuse; 1263 break; 1264 1265 case SO_REUSEPORT: 1266 v.val = sk->sk_reuseport; 1267 break; 1268 1269 case SO_KEEPALIVE: 1270 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1271 break; 1272 1273 case SO_TYPE: 1274 v.val = sk->sk_type; 1275 break; 1276 1277 case SO_PROTOCOL: 1278 v.val = sk->sk_protocol; 1279 break; 1280 1281 case SO_DOMAIN: 1282 v.val = sk->sk_family; 1283 break; 1284 1285 case SO_ERROR: 1286 v.val = -sock_error(sk); 1287 if (v.val == 0) 1288 v.val = xchg(&sk->sk_err_soft, 0); 1289 break; 1290 1291 case SO_OOBINLINE: 1292 v.val = sock_flag(sk, SOCK_URGINLINE); 1293 break; 1294 1295 case SO_NO_CHECK: 1296 v.val = sk->sk_no_check_tx; 1297 break; 1298 1299 case SO_PRIORITY: 1300 v.val = sk->sk_priority; 1301 break; 1302 1303 case SO_LINGER: 1304 lv = sizeof(v.ling); 1305 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1306 v.ling.l_linger = sk->sk_lingertime / HZ; 1307 break; 1308 1309 case SO_BSDCOMPAT: 1310 sock_warn_obsolete_bsdism("getsockopt"); 1311 break; 1312 1313 case SO_TIMESTAMP_OLD: 1314 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1315 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1316 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1317 break; 1318 1319 case SO_TIMESTAMPNS_OLD: 1320 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1321 break; 1322 1323 case SO_TIMESTAMP_NEW: 1324 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1325 break; 1326 1327 case SO_TIMESTAMPNS_NEW: 1328 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1329 break; 1330 1331 case SO_TIMESTAMPING_OLD: 1332 v.val = sk->sk_tsflags; 1333 break; 1334 1335 case SO_RCVTIMEO_OLD: 1336 case SO_RCVTIMEO_NEW: 1337 lv = sock_get_timeout(sk->sk_rcvtimeo, &v, SO_RCVTIMEO_OLD == optname); 1338 break; 1339 1340 case SO_SNDTIMEO_OLD: 1341 case SO_SNDTIMEO_NEW: 1342 lv = sock_get_timeout(sk->sk_sndtimeo, &v, SO_SNDTIMEO_OLD == optname); 1343 break; 1344 1345 case SO_RCVLOWAT: 1346 v.val = sk->sk_rcvlowat; 1347 break; 1348 1349 case SO_SNDLOWAT: 1350 v.val = 1; 1351 break; 1352 1353 case SO_PASSCRED: 1354 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1355 break; 1356 1357 case SO_PEERCRED: 1358 { 1359 struct ucred peercred; 1360 if (len > sizeof(peercred)) 1361 len = sizeof(peercred); 1362 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1363 if (copy_to_user(optval, &peercred, len)) 1364 return -EFAULT; 1365 goto lenout; 1366 } 1367 1368 case SO_PEERGROUPS: 1369 { 1370 int ret, n; 1371 1372 if (!sk->sk_peer_cred) 1373 return -ENODATA; 1374 1375 n = sk->sk_peer_cred->group_info->ngroups; 1376 if (len < n * sizeof(gid_t)) { 1377 len = n * sizeof(gid_t); 1378 return put_user(len, optlen) ? -EFAULT : -ERANGE; 1379 } 1380 len = n * sizeof(gid_t); 1381 1382 ret = groups_to_user((gid_t __user *)optval, 1383 sk->sk_peer_cred->group_info); 1384 if (ret) 1385 return ret; 1386 goto lenout; 1387 } 1388 1389 case SO_PEERNAME: 1390 { 1391 char address[128]; 1392 1393 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2); 1394 if (lv < 0) 1395 return -ENOTCONN; 1396 if (lv < len) 1397 return -EINVAL; 1398 if (copy_to_user(optval, address, len)) 1399 return -EFAULT; 1400 goto lenout; 1401 } 1402 1403 /* Dubious BSD thing... Probably nobody even uses it, but 1404 * the UNIX standard wants it for whatever reason... -DaveM 1405 */ 1406 case SO_ACCEPTCONN: 1407 v.val = sk->sk_state == TCP_LISTEN; 1408 break; 1409 1410 case SO_PASSSEC: 1411 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1412 break; 1413 1414 case SO_PEERSEC: 1415 return security_socket_getpeersec_stream(sock, optval, optlen, len); 1416 1417 case SO_MARK: 1418 v.val = sk->sk_mark; 1419 break; 1420 1421 case SO_RXQ_OVFL: 1422 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1423 break; 1424 1425 case SO_WIFI_STATUS: 1426 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1427 break; 1428 1429 case SO_PEEK_OFF: 1430 if (!sock->ops->set_peek_off) 1431 return -EOPNOTSUPP; 1432 1433 v.val = sk->sk_peek_off; 1434 break; 1435 case SO_NOFCS: 1436 v.val = sock_flag(sk, SOCK_NOFCS); 1437 break; 1438 1439 case SO_BINDTODEVICE: 1440 return sock_getbindtodevice(sk, optval, optlen, len); 1441 1442 case SO_GET_FILTER: 1443 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len); 1444 if (len < 0) 1445 return len; 1446 1447 goto lenout; 1448 1449 case SO_LOCK_FILTER: 1450 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1451 break; 1452 1453 case SO_BPF_EXTENSIONS: 1454 v.val = bpf_tell_extensions(); 1455 break; 1456 1457 case SO_SELECT_ERR_QUEUE: 1458 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1459 break; 1460 1461 #ifdef CONFIG_NET_RX_BUSY_POLL 1462 case SO_BUSY_POLL: 1463 v.val = sk->sk_ll_usec; 1464 break; 1465 #endif 1466 1467 case SO_MAX_PACING_RATE: 1468 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 1469 lv = sizeof(v.ulval); 1470 v.ulval = sk->sk_max_pacing_rate; 1471 } else { 1472 /* 32bit version */ 1473 v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U); 1474 } 1475 break; 1476 1477 case SO_INCOMING_CPU: 1478 v.val = sk->sk_incoming_cpu; 1479 break; 1480 1481 case SO_MEMINFO: 1482 { 1483 u32 meminfo[SK_MEMINFO_VARS]; 1484 1485 if (get_user(len, optlen)) 1486 return -EFAULT; 1487 1488 sk_get_meminfo(sk, meminfo); 1489 1490 len = min_t(unsigned int, len, sizeof(meminfo)); 1491 if (copy_to_user(optval, &meminfo, len)) 1492 return -EFAULT; 1493 1494 goto lenout; 1495 } 1496 1497 #ifdef CONFIG_NET_RX_BUSY_POLL 1498 case SO_INCOMING_NAPI_ID: 1499 v.val = READ_ONCE(sk->sk_napi_id); 1500 1501 /* aggregate non-NAPI IDs down to 0 */ 1502 if (v.val < MIN_NAPI_ID) 1503 v.val = 0; 1504 1505 break; 1506 #endif 1507 1508 case SO_COOKIE: 1509 lv = sizeof(u64); 1510 if (len < lv) 1511 return -EINVAL; 1512 v.val64 = sock_gen_cookie(sk); 1513 break; 1514 1515 case SO_ZEROCOPY: 1516 v.val = sock_flag(sk, SOCK_ZEROCOPY); 1517 break; 1518 1519 case SO_TXTIME: 1520 lv = sizeof(v.txtime); 1521 v.txtime.clockid = sk->sk_clockid; 1522 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 1523 SOF_TXTIME_DEADLINE_MODE : 0; 1524 v.txtime.flags |= sk->sk_txtime_report_errors ? 1525 SOF_TXTIME_REPORT_ERRORS : 0; 1526 break; 1527 1528 case SO_BINDTOIFINDEX: 1529 v.val = sk->sk_bound_dev_if; 1530 break; 1531 1532 default: 1533 /* We implement the SO_SNDLOWAT etc to not be settable 1534 * (1003.1g 7). 1535 */ 1536 return -ENOPROTOOPT; 1537 } 1538 1539 if (len > lv) 1540 len = lv; 1541 if (copy_to_user(optval, &v, len)) 1542 return -EFAULT; 1543 lenout: 1544 if (put_user(len, optlen)) 1545 return -EFAULT; 1546 return 0; 1547 } 1548 1549 /* 1550 * Initialize an sk_lock. 1551 * 1552 * (We also register the sk_lock with the lock validator.) 1553 */ 1554 static inline void sock_lock_init(struct sock *sk) 1555 { 1556 if (sk->sk_kern_sock) 1557 sock_lock_init_class_and_name( 1558 sk, 1559 af_family_kern_slock_key_strings[sk->sk_family], 1560 af_family_kern_slock_keys + sk->sk_family, 1561 af_family_kern_key_strings[sk->sk_family], 1562 af_family_kern_keys + sk->sk_family); 1563 else 1564 sock_lock_init_class_and_name( 1565 sk, 1566 af_family_slock_key_strings[sk->sk_family], 1567 af_family_slock_keys + sk->sk_family, 1568 af_family_key_strings[sk->sk_family], 1569 af_family_keys + sk->sk_family); 1570 } 1571 1572 /* 1573 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 1574 * even temporarly, because of RCU lookups. sk_node should also be left as is. 1575 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 1576 */ 1577 static void sock_copy(struct sock *nsk, const struct sock *osk) 1578 { 1579 #ifdef CONFIG_SECURITY_NETWORK 1580 void *sptr = nsk->sk_security; 1581 #endif 1582 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 1583 1584 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 1585 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 1586 1587 #ifdef CONFIG_SECURITY_NETWORK 1588 nsk->sk_security = sptr; 1589 security_sk_clone(osk, nsk); 1590 #endif 1591 } 1592 1593 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 1594 int family) 1595 { 1596 struct sock *sk; 1597 struct kmem_cache *slab; 1598 1599 slab = prot->slab; 1600 if (slab != NULL) { 1601 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 1602 if (!sk) 1603 return sk; 1604 if (priority & __GFP_ZERO) 1605 sk_prot_clear_nulls(sk, prot->obj_size); 1606 } else 1607 sk = kmalloc(prot->obj_size, priority); 1608 1609 if (sk != NULL) { 1610 if (security_sk_alloc(sk, family, priority)) 1611 goto out_free; 1612 1613 if (!try_module_get(prot->owner)) 1614 goto out_free_sec; 1615 sk_tx_queue_clear(sk); 1616 } 1617 1618 return sk; 1619 1620 out_free_sec: 1621 security_sk_free(sk); 1622 out_free: 1623 if (slab != NULL) 1624 kmem_cache_free(slab, sk); 1625 else 1626 kfree(sk); 1627 return NULL; 1628 } 1629 1630 static void sk_prot_free(struct proto *prot, struct sock *sk) 1631 { 1632 struct kmem_cache *slab; 1633 struct module *owner; 1634 1635 owner = prot->owner; 1636 slab = prot->slab; 1637 1638 cgroup_sk_free(&sk->sk_cgrp_data); 1639 mem_cgroup_sk_free(sk); 1640 security_sk_free(sk); 1641 if (slab != NULL) 1642 kmem_cache_free(slab, sk); 1643 else 1644 kfree(sk); 1645 module_put(owner); 1646 } 1647 1648 /** 1649 * sk_alloc - All socket objects are allocated here 1650 * @net: the applicable net namespace 1651 * @family: protocol family 1652 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1653 * @prot: struct proto associated with this new sock instance 1654 * @kern: is this to be a kernel socket? 1655 */ 1656 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 1657 struct proto *prot, int kern) 1658 { 1659 struct sock *sk; 1660 1661 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 1662 if (sk) { 1663 sk->sk_family = family; 1664 /* 1665 * See comment in struct sock definition to understand 1666 * why we need sk_prot_creator -acme 1667 */ 1668 sk->sk_prot = sk->sk_prot_creator = prot; 1669 sk->sk_kern_sock = kern; 1670 sock_lock_init(sk); 1671 sk->sk_net_refcnt = kern ? 0 : 1; 1672 if (likely(sk->sk_net_refcnt)) { 1673 get_net(net); 1674 sock_inuse_add(net, 1); 1675 } 1676 1677 sock_net_set(sk, net); 1678 refcount_set(&sk->sk_wmem_alloc, 1); 1679 1680 mem_cgroup_sk_alloc(sk); 1681 cgroup_sk_alloc(&sk->sk_cgrp_data); 1682 sock_update_classid(&sk->sk_cgrp_data); 1683 sock_update_netprioidx(&sk->sk_cgrp_data); 1684 } 1685 1686 return sk; 1687 } 1688 EXPORT_SYMBOL(sk_alloc); 1689 1690 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 1691 * grace period. This is the case for UDP sockets and TCP listeners. 1692 */ 1693 static void __sk_destruct(struct rcu_head *head) 1694 { 1695 struct sock *sk = container_of(head, struct sock, sk_rcu); 1696 struct sk_filter *filter; 1697 1698 if (sk->sk_destruct) 1699 sk->sk_destruct(sk); 1700 1701 filter = rcu_dereference_check(sk->sk_filter, 1702 refcount_read(&sk->sk_wmem_alloc) == 0); 1703 if (filter) { 1704 sk_filter_uncharge(sk, filter); 1705 RCU_INIT_POINTER(sk->sk_filter, NULL); 1706 } 1707 if (rcu_access_pointer(sk->sk_reuseport_cb)) 1708 reuseport_detach_sock(sk); 1709 1710 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 1711 1712 if (atomic_read(&sk->sk_omem_alloc)) 1713 pr_debug("%s: optmem leakage (%d bytes) detected\n", 1714 __func__, atomic_read(&sk->sk_omem_alloc)); 1715 1716 if (sk->sk_frag.page) { 1717 put_page(sk->sk_frag.page); 1718 sk->sk_frag.page = NULL; 1719 } 1720 1721 if (sk->sk_peer_cred) 1722 put_cred(sk->sk_peer_cred); 1723 put_pid(sk->sk_peer_pid); 1724 if (likely(sk->sk_net_refcnt)) 1725 put_net(sock_net(sk)); 1726 sk_prot_free(sk->sk_prot_creator, sk); 1727 } 1728 1729 void sk_destruct(struct sock *sk) 1730 { 1731 if (sock_flag(sk, SOCK_RCU_FREE)) 1732 call_rcu(&sk->sk_rcu, __sk_destruct); 1733 else 1734 __sk_destruct(&sk->sk_rcu); 1735 } 1736 1737 static void __sk_free(struct sock *sk) 1738 { 1739 if (likely(sk->sk_net_refcnt)) 1740 sock_inuse_add(sock_net(sk), -1); 1741 1742 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 1743 sock_diag_broadcast_destroy(sk); 1744 else 1745 sk_destruct(sk); 1746 } 1747 1748 void sk_free(struct sock *sk) 1749 { 1750 /* 1751 * We subtract one from sk_wmem_alloc and can know if 1752 * some packets are still in some tx queue. 1753 * If not null, sock_wfree() will call __sk_free(sk) later 1754 */ 1755 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 1756 __sk_free(sk); 1757 } 1758 EXPORT_SYMBOL(sk_free); 1759 1760 static void sk_init_common(struct sock *sk) 1761 { 1762 skb_queue_head_init(&sk->sk_receive_queue); 1763 skb_queue_head_init(&sk->sk_write_queue); 1764 skb_queue_head_init(&sk->sk_error_queue); 1765 1766 rwlock_init(&sk->sk_callback_lock); 1767 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 1768 af_rlock_keys + sk->sk_family, 1769 af_family_rlock_key_strings[sk->sk_family]); 1770 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 1771 af_wlock_keys + sk->sk_family, 1772 af_family_wlock_key_strings[sk->sk_family]); 1773 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 1774 af_elock_keys + sk->sk_family, 1775 af_family_elock_key_strings[sk->sk_family]); 1776 lockdep_set_class_and_name(&sk->sk_callback_lock, 1777 af_callback_keys + sk->sk_family, 1778 af_family_clock_key_strings[sk->sk_family]); 1779 } 1780 1781 /** 1782 * sk_clone_lock - clone a socket, and lock its clone 1783 * @sk: the socket to clone 1784 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1785 * 1786 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1787 */ 1788 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 1789 { 1790 struct sock *newsk; 1791 bool is_charged = true; 1792 1793 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family); 1794 if (newsk != NULL) { 1795 struct sk_filter *filter; 1796 1797 sock_copy(newsk, sk); 1798 1799 newsk->sk_prot_creator = sk->sk_prot; 1800 1801 /* SANITY */ 1802 if (likely(newsk->sk_net_refcnt)) 1803 get_net(sock_net(newsk)); 1804 sk_node_init(&newsk->sk_node); 1805 sock_lock_init(newsk); 1806 bh_lock_sock(newsk); 1807 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 1808 newsk->sk_backlog.len = 0; 1809 1810 atomic_set(&newsk->sk_rmem_alloc, 0); 1811 /* 1812 * sk_wmem_alloc set to one (see sk_free() and sock_wfree()) 1813 */ 1814 refcount_set(&newsk->sk_wmem_alloc, 1); 1815 atomic_set(&newsk->sk_omem_alloc, 0); 1816 sk_init_common(newsk); 1817 1818 newsk->sk_dst_cache = NULL; 1819 newsk->sk_dst_pending_confirm = 0; 1820 newsk->sk_wmem_queued = 0; 1821 newsk->sk_forward_alloc = 0; 1822 atomic_set(&newsk->sk_drops, 0); 1823 newsk->sk_send_head = NULL; 1824 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 1825 atomic_set(&newsk->sk_zckey, 0); 1826 1827 sock_reset_flag(newsk, SOCK_DONE); 1828 mem_cgroup_sk_alloc(newsk); 1829 cgroup_sk_alloc(&newsk->sk_cgrp_data); 1830 1831 rcu_read_lock(); 1832 filter = rcu_dereference(sk->sk_filter); 1833 if (filter != NULL) 1834 /* though it's an empty new sock, the charging may fail 1835 * if sysctl_optmem_max was changed between creation of 1836 * original socket and cloning 1837 */ 1838 is_charged = sk_filter_charge(newsk, filter); 1839 RCU_INIT_POINTER(newsk->sk_filter, filter); 1840 rcu_read_unlock(); 1841 1842 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 1843 /* We need to make sure that we don't uncharge the new 1844 * socket if we couldn't charge it in the first place 1845 * as otherwise we uncharge the parent's filter. 1846 */ 1847 if (!is_charged) 1848 RCU_INIT_POINTER(newsk->sk_filter, NULL); 1849 sk_free_unlock_clone(newsk); 1850 newsk = NULL; 1851 goto out; 1852 } 1853 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 1854 1855 newsk->sk_err = 0; 1856 newsk->sk_err_soft = 0; 1857 newsk->sk_priority = 0; 1858 newsk->sk_incoming_cpu = raw_smp_processor_id(); 1859 if (likely(newsk->sk_net_refcnt)) 1860 sock_inuse_add(sock_net(newsk), 1); 1861 1862 /* 1863 * Before updating sk_refcnt, we must commit prior changes to memory 1864 * (Documentation/RCU/rculist_nulls.txt for details) 1865 */ 1866 smp_wmb(); 1867 refcount_set(&newsk->sk_refcnt, 2); 1868 1869 /* 1870 * Increment the counter in the same struct proto as the master 1871 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that 1872 * is the same as sk->sk_prot->socks, as this field was copied 1873 * with memcpy). 1874 * 1875 * This _changes_ the previous behaviour, where 1876 * tcp_create_openreq_child always was incrementing the 1877 * equivalent to tcp_prot->socks (inet_sock_nr), so this have 1878 * to be taken into account in all callers. -acme 1879 */ 1880 sk_refcnt_debug_inc(newsk); 1881 sk_set_socket(newsk, NULL); 1882 RCU_INIT_POINTER(newsk->sk_wq, NULL); 1883 1884 if (newsk->sk_prot->sockets_allocated) 1885 sk_sockets_allocated_inc(newsk); 1886 1887 if (sock_needs_netstamp(sk) && 1888 newsk->sk_flags & SK_FLAGS_TIMESTAMP) 1889 net_enable_timestamp(); 1890 } 1891 out: 1892 return newsk; 1893 } 1894 EXPORT_SYMBOL_GPL(sk_clone_lock); 1895 1896 void sk_free_unlock_clone(struct sock *sk) 1897 { 1898 /* It is still raw copy of parent, so invalidate 1899 * destructor and make plain sk_free() */ 1900 sk->sk_destruct = NULL; 1901 bh_unlock_sock(sk); 1902 sk_free(sk); 1903 } 1904 EXPORT_SYMBOL_GPL(sk_free_unlock_clone); 1905 1906 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 1907 { 1908 u32 max_segs = 1; 1909 1910 sk_dst_set(sk, dst); 1911 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps; 1912 if (sk->sk_route_caps & NETIF_F_GSO) 1913 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 1914 sk->sk_route_caps &= ~sk->sk_route_nocaps; 1915 if (sk_can_gso(sk)) { 1916 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 1917 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 1918 } else { 1919 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 1920 sk->sk_gso_max_size = dst->dev->gso_max_size; 1921 max_segs = max_t(u32, dst->dev->gso_max_segs, 1); 1922 } 1923 } 1924 sk->sk_gso_max_segs = max_segs; 1925 } 1926 EXPORT_SYMBOL_GPL(sk_setup_caps); 1927 1928 /* 1929 * Simple resource managers for sockets. 1930 */ 1931 1932 1933 /* 1934 * Write buffer destructor automatically called from kfree_skb. 1935 */ 1936 void sock_wfree(struct sk_buff *skb) 1937 { 1938 struct sock *sk = skb->sk; 1939 unsigned int len = skb->truesize; 1940 1941 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 1942 /* 1943 * Keep a reference on sk_wmem_alloc, this will be released 1944 * after sk_write_space() call 1945 */ 1946 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 1947 sk->sk_write_space(sk); 1948 len = 1; 1949 } 1950 /* 1951 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 1952 * could not do because of in-flight packets 1953 */ 1954 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 1955 __sk_free(sk); 1956 } 1957 EXPORT_SYMBOL(sock_wfree); 1958 1959 /* This variant of sock_wfree() is used by TCP, 1960 * since it sets SOCK_USE_WRITE_QUEUE. 1961 */ 1962 void __sock_wfree(struct sk_buff *skb) 1963 { 1964 struct sock *sk = skb->sk; 1965 1966 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 1967 __sk_free(sk); 1968 } 1969 1970 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 1971 { 1972 skb_orphan(skb); 1973 skb->sk = sk; 1974 #ifdef CONFIG_INET 1975 if (unlikely(!sk_fullsock(sk))) { 1976 skb->destructor = sock_edemux; 1977 sock_hold(sk); 1978 return; 1979 } 1980 #endif 1981 skb->destructor = sock_wfree; 1982 skb_set_hash_from_sk(skb, sk); 1983 /* 1984 * We used to take a refcount on sk, but following operation 1985 * is enough to guarantee sk_free() wont free this sock until 1986 * all in-flight packets are completed 1987 */ 1988 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 1989 } 1990 EXPORT_SYMBOL(skb_set_owner_w); 1991 1992 /* This helper is used by netem, as it can hold packets in its 1993 * delay queue. We want to allow the owner socket to send more 1994 * packets, as if they were already TX completed by a typical driver. 1995 * But we also want to keep skb->sk set because some packet schedulers 1996 * rely on it (sch_fq for example). 1997 */ 1998 void skb_orphan_partial(struct sk_buff *skb) 1999 { 2000 if (skb_is_tcp_pure_ack(skb)) 2001 return; 2002 2003 if (skb->destructor == sock_wfree 2004 #ifdef CONFIG_INET 2005 || skb->destructor == tcp_wfree 2006 #endif 2007 ) { 2008 struct sock *sk = skb->sk; 2009 2010 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 2011 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)); 2012 skb->destructor = sock_efree; 2013 } 2014 } else { 2015 skb_orphan(skb); 2016 } 2017 } 2018 EXPORT_SYMBOL(skb_orphan_partial); 2019 2020 /* 2021 * Read buffer destructor automatically called from kfree_skb. 2022 */ 2023 void sock_rfree(struct sk_buff *skb) 2024 { 2025 struct sock *sk = skb->sk; 2026 unsigned int len = skb->truesize; 2027 2028 atomic_sub(len, &sk->sk_rmem_alloc); 2029 sk_mem_uncharge(sk, len); 2030 } 2031 EXPORT_SYMBOL(sock_rfree); 2032 2033 /* 2034 * Buffer destructor for skbs that are not used directly in read or write 2035 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2036 */ 2037 void sock_efree(struct sk_buff *skb) 2038 { 2039 sock_put(skb->sk); 2040 } 2041 EXPORT_SYMBOL(sock_efree); 2042 2043 kuid_t sock_i_uid(struct sock *sk) 2044 { 2045 kuid_t uid; 2046 2047 read_lock_bh(&sk->sk_callback_lock); 2048 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 2049 read_unlock_bh(&sk->sk_callback_lock); 2050 return uid; 2051 } 2052 EXPORT_SYMBOL(sock_i_uid); 2053 2054 unsigned long sock_i_ino(struct sock *sk) 2055 { 2056 unsigned long ino; 2057 2058 read_lock_bh(&sk->sk_callback_lock); 2059 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 2060 read_unlock_bh(&sk->sk_callback_lock); 2061 return ino; 2062 } 2063 EXPORT_SYMBOL(sock_i_ino); 2064 2065 /* 2066 * Allocate a skb from the socket's send buffer. 2067 */ 2068 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2069 gfp_t priority) 2070 { 2071 if (force || refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) { 2072 struct sk_buff *skb = alloc_skb(size, priority); 2073 if (skb) { 2074 skb_set_owner_w(skb, sk); 2075 return skb; 2076 } 2077 } 2078 return NULL; 2079 } 2080 EXPORT_SYMBOL(sock_wmalloc); 2081 2082 static void sock_ofree(struct sk_buff *skb) 2083 { 2084 struct sock *sk = skb->sk; 2085 2086 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2087 } 2088 2089 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2090 gfp_t priority) 2091 { 2092 struct sk_buff *skb; 2093 2094 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2095 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2096 sysctl_optmem_max) 2097 return NULL; 2098 2099 skb = alloc_skb(size, priority); 2100 if (!skb) 2101 return NULL; 2102 2103 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2104 skb->sk = sk; 2105 skb->destructor = sock_ofree; 2106 return skb; 2107 } 2108 2109 /* 2110 * Allocate a memory block from the socket's option memory buffer. 2111 */ 2112 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2113 { 2114 if ((unsigned int)size <= sysctl_optmem_max && 2115 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { 2116 void *mem; 2117 /* First do the add, to avoid the race if kmalloc 2118 * might sleep. 2119 */ 2120 atomic_add(size, &sk->sk_omem_alloc); 2121 mem = kmalloc(size, priority); 2122 if (mem) 2123 return mem; 2124 atomic_sub(size, &sk->sk_omem_alloc); 2125 } 2126 return NULL; 2127 } 2128 EXPORT_SYMBOL(sock_kmalloc); 2129 2130 /* Free an option memory block. Note, we actually want the inline 2131 * here as this allows gcc to detect the nullify and fold away the 2132 * condition entirely. 2133 */ 2134 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2135 const bool nullify) 2136 { 2137 if (WARN_ON_ONCE(!mem)) 2138 return; 2139 if (nullify) 2140 kzfree(mem); 2141 else 2142 kfree(mem); 2143 atomic_sub(size, &sk->sk_omem_alloc); 2144 } 2145 2146 void sock_kfree_s(struct sock *sk, void *mem, int size) 2147 { 2148 __sock_kfree_s(sk, mem, size, false); 2149 } 2150 EXPORT_SYMBOL(sock_kfree_s); 2151 2152 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2153 { 2154 __sock_kfree_s(sk, mem, size, true); 2155 } 2156 EXPORT_SYMBOL(sock_kzfree_s); 2157 2158 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2159 I think, these locks should be removed for datagram sockets. 2160 */ 2161 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2162 { 2163 DEFINE_WAIT(wait); 2164 2165 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2166 for (;;) { 2167 if (!timeo) 2168 break; 2169 if (signal_pending(current)) 2170 break; 2171 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2172 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2173 if (refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) 2174 break; 2175 if (sk->sk_shutdown & SEND_SHUTDOWN) 2176 break; 2177 if (sk->sk_err) 2178 break; 2179 timeo = schedule_timeout(timeo); 2180 } 2181 finish_wait(sk_sleep(sk), &wait); 2182 return timeo; 2183 } 2184 2185 2186 /* 2187 * Generic send/receive buffer handlers 2188 */ 2189 2190 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2191 unsigned long data_len, int noblock, 2192 int *errcode, int max_page_order) 2193 { 2194 struct sk_buff *skb; 2195 long timeo; 2196 int err; 2197 2198 timeo = sock_sndtimeo(sk, noblock); 2199 for (;;) { 2200 err = sock_error(sk); 2201 if (err != 0) 2202 goto failure; 2203 2204 err = -EPIPE; 2205 if (sk->sk_shutdown & SEND_SHUTDOWN) 2206 goto failure; 2207 2208 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf) 2209 break; 2210 2211 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2212 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2213 err = -EAGAIN; 2214 if (!timeo) 2215 goto failure; 2216 if (signal_pending(current)) 2217 goto interrupted; 2218 timeo = sock_wait_for_wmem(sk, timeo); 2219 } 2220 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 2221 errcode, sk->sk_allocation); 2222 if (skb) 2223 skb_set_owner_w(skb, sk); 2224 return skb; 2225 2226 interrupted: 2227 err = sock_intr_errno(timeo); 2228 failure: 2229 *errcode = err; 2230 return NULL; 2231 } 2232 EXPORT_SYMBOL(sock_alloc_send_pskb); 2233 2234 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size, 2235 int noblock, int *errcode) 2236 { 2237 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0); 2238 } 2239 EXPORT_SYMBOL(sock_alloc_send_skb); 2240 2241 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, 2242 struct sockcm_cookie *sockc) 2243 { 2244 u32 tsflags; 2245 2246 switch (cmsg->cmsg_type) { 2247 case SO_MARK: 2248 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 2249 return -EPERM; 2250 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2251 return -EINVAL; 2252 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 2253 break; 2254 case SO_TIMESTAMPING_OLD: 2255 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2256 return -EINVAL; 2257 2258 tsflags = *(u32 *)CMSG_DATA(cmsg); 2259 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 2260 return -EINVAL; 2261 2262 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 2263 sockc->tsflags |= tsflags; 2264 break; 2265 case SCM_TXTIME: 2266 if (!sock_flag(sk, SOCK_TXTIME)) 2267 return -EINVAL; 2268 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 2269 return -EINVAL; 2270 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); 2271 break; 2272 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 2273 case SCM_RIGHTS: 2274 case SCM_CREDENTIALS: 2275 break; 2276 default: 2277 return -EINVAL; 2278 } 2279 return 0; 2280 } 2281 EXPORT_SYMBOL(__sock_cmsg_send); 2282 2283 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 2284 struct sockcm_cookie *sockc) 2285 { 2286 struct cmsghdr *cmsg; 2287 int ret; 2288 2289 for_each_cmsghdr(cmsg, msg) { 2290 if (!CMSG_OK(msg, cmsg)) 2291 return -EINVAL; 2292 if (cmsg->cmsg_level != SOL_SOCKET) 2293 continue; 2294 ret = __sock_cmsg_send(sk, msg, cmsg, sockc); 2295 if (ret) 2296 return ret; 2297 } 2298 return 0; 2299 } 2300 EXPORT_SYMBOL(sock_cmsg_send); 2301 2302 static void sk_enter_memory_pressure(struct sock *sk) 2303 { 2304 if (!sk->sk_prot->enter_memory_pressure) 2305 return; 2306 2307 sk->sk_prot->enter_memory_pressure(sk); 2308 } 2309 2310 static void sk_leave_memory_pressure(struct sock *sk) 2311 { 2312 if (sk->sk_prot->leave_memory_pressure) { 2313 sk->sk_prot->leave_memory_pressure(sk); 2314 } else { 2315 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 2316 2317 if (memory_pressure && *memory_pressure) 2318 *memory_pressure = 0; 2319 } 2320 } 2321 2322 /* On 32bit arches, an skb frag is limited to 2^15 */ 2323 #define SKB_FRAG_PAGE_ORDER get_order(32768) 2324 2325 /** 2326 * skb_page_frag_refill - check that a page_frag contains enough room 2327 * @sz: minimum size of the fragment we want to get 2328 * @pfrag: pointer to page_frag 2329 * @gfp: priority for memory allocation 2330 * 2331 * Note: While this allocator tries to use high order pages, there is 2332 * no guarantee that allocations succeed. Therefore, @sz MUST be 2333 * less or equal than PAGE_SIZE. 2334 */ 2335 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 2336 { 2337 if (pfrag->page) { 2338 if (page_ref_count(pfrag->page) == 1) { 2339 pfrag->offset = 0; 2340 return true; 2341 } 2342 if (pfrag->offset + sz <= pfrag->size) 2343 return true; 2344 put_page(pfrag->page); 2345 } 2346 2347 pfrag->offset = 0; 2348 if (SKB_FRAG_PAGE_ORDER) { 2349 /* Avoid direct reclaim but allow kswapd to wake */ 2350 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 2351 __GFP_COMP | __GFP_NOWARN | 2352 __GFP_NORETRY, 2353 SKB_FRAG_PAGE_ORDER); 2354 if (likely(pfrag->page)) { 2355 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 2356 return true; 2357 } 2358 } 2359 pfrag->page = alloc_page(gfp); 2360 if (likely(pfrag->page)) { 2361 pfrag->size = PAGE_SIZE; 2362 return true; 2363 } 2364 return false; 2365 } 2366 EXPORT_SYMBOL(skb_page_frag_refill); 2367 2368 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2369 { 2370 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2371 return true; 2372 2373 sk_enter_memory_pressure(sk); 2374 sk_stream_moderate_sndbuf(sk); 2375 return false; 2376 } 2377 EXPORT_SYMBOL(sk_page_frag_refill); 2378 2379 static void __lock_sock(struct sock *sk) 2380 __releases(&sk->sk_lock.slock) 2381 __acquires(&sk->sk_lock.slock) 2382 { 2383 DEFINE_WAIT(wait); 2384 2385 for (;;) { 2386 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2387 TASK_UNINTERRUPTIBLE); 2388 spin_unlock_bh(&sk->sk_lock.slock); 2389 schedule(); 2390 spin_lock_bh(&sk->sk_lock.slock); 2391 if (!sock_owned_by_user(sk)) 2392 break; 2393 } 2394 finish_wait(&sk->sk_lock.wq, &wait); 2395 } 2396 2397 void __release_sock(struct sock *sk) 2398 __releases(&sk->sk_lock.slock) 2399 __acquires(&sk->sk_lock.slock) 2400 { 2401 struct sk_buff *skb, *next; 2402 2403 while ((skb = sk->sk_backlog.head) != NULL) { 2404 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2405 2406 spin_unlock_bh(&sk->sk_lock.slock); 2407 2408 do { 2409 next = skb->next; 2410 prefetch(next); 2411 WARN_ON_ONCE(skb_dst_is_noref(skb)); 2412 skb_mark_not_on_list(skb); 2413 sk_backlog_rcv(sk, skb); 2414 2415 cond_resched(); 2416 2417 skb = next; 2418 } while (skb != NULL); 2419 2420 spin_lock_bh(&sk->sk_lock.slock); 2421 } 2422 2423 /* 2424 * Doing the zeroing here guarantee we can not loop forever 2425 * while a wild producer attempts to flood us. 2426 */ 2427 sk->sk_backlog.len = 0; 2428 } 2429 2430 void __sk_flush_backlog(struct sock *sk) 2431 { 2432 spin_lock_bh(&sk->sk_lock.slock); 2433 __release_sock(sk); 2434 spin_unlock_bh(&sk->sk_lock.slock); 2435 } 2436 2437 /** 2438 * sk_wait_data - wait for data to arrive at sk_receive_queue 2439 * @sk: sock to wait on 2440 * @timeo: for how long 2441 * @skb: last skb seen on sk_receive_queue 2442 * 2443 * Now socket state including sk->sk_err is changed only under lock, 2444 * hence we may omit checks after joining wait queue. 2445 * We check receive queue before schedule() only as optimization; 2446 * it is very likely that release_sock() added new data. 2447 */ 2448 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 2449 { 2450 DEFINE_WAIT_FUNC(wait, woken_wake_function); 2451 int rc; 2452 2453 add_wait_queue(sk_sleep(sk), &wait); 2454 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2455 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 2456 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 2457 remove_wait_queue(sk_sleep(sk), &wait); 2458 return rc; 2459 } 2460 EXPORT_SYMBOL(sk_wait_data); 2461 2462 /** 2463 * __sk_mem_raise_allocated - increase memory_allocated 2464 * @sk: socket 2465 * @size: memory size to allocate 2466 * @amt: pages to allocate 2467 * @kind: allocation type 2468 * 2469 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc 2470 */ 2471 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 2472 { 2473 struct proto *prot = sk->sk_prot; 2474 long allocated = sk_memory_allocated_add(sk, amt); 2475 bool charged = true; 2476 2477 if (mem_cgroup_sockets_enabled && sk->sk_memcg && 2478 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt))) 2479 goto suppress_allocation; 2480 2481 /* Under limit. */ 2482 if (allocated <= sk_prot_mem_limits(sk, 0)) { 2483 sk_leave_memory_pressure(sk); 2484 return 1; 2485 } 2486 2487 /* Under pressure. */ 2488 if (allocated > sk_prot_mem_limits(sk, 1)) 2489 sk_enter_memory_pressure(sk); 2490 2491 /* Over hard limit. */ 2492 if (allocated > sk_prot_mem_limits(sk, 2)) 2493 goto suppress_allocation; 2494 2495 /* guarantee minimum buffer size under pressure */ 2496 if (kind == SK_MEM_RECV) { 2497 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 2498 return 1; 2499 2500 } else { /* SK_MEM_SEND */ 2501 int wmem0 = sk_get_wmem0(sk, prot); 2502 2503 if (sk->sk_type == SOCK_STREAM) { 2504 if (sk->sk_wmem_queued < wmem0) 2505 return 1; 2506 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 2507 return 1; 2508 } 2509 } 2510 2511 if (sk_has_memory_pressure(sk)) { 2512 u64 alloc; 2513 2514 if (!sk_under_memory_pressure(sk)) 2515 return 1; 2516 alloc = sk_sockets_allocated_read_positive(sk); 2517 if (sk_prot_mem_limits(sk, 2) > alloc * 2518 sk_mem_pages(sk->sk_wmem_queued + 2519 atomic_read(&sk->sk_rmem_alloc) + 2520 sk->sk_forward_alloc)) 2521 return 1; 2522 } 2523 2524 suppress_allocation: 2525 2526 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 2527 sk_stream_moderate_sndbuf(sk); 2528 2529 /* Fail only if socket is _under_ its sndbuf. 2530 * In this case we cannot block, so that we have to fail. 2531 */ 2532 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) 2533 return 1; 2534 } 2535 2536 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged)) 2537 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 2538 2539 sk_memory_allocated_sub(sk, amt); 2540 2541 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2542 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt); 2543 2544 return 0; 2545 } 2546 EXPORT_SYMBOL(__sk_mem_raise_allocated); 2547 2548 /** 2549 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 2550 * @sk: socket 2551 * @size: memory size to allocate 2552 * @kind: allocation type 2553 * 2554 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 2555 * rmem allocation. This function assumes that protocols which have 2556 * memory_pressure use sk_wmem_queued as write buffer accounting. 2557 */ 2558 int __sk_mem_schedule(struct sock *sk, int size, int kind) 2559 { 2560 int ret, amt = sk_mem_pages(size); 2561 2562 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT; 2563 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 2564 if (!ret) 2565 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT; 2566 return ret; 2567 } 2568 EXPORT_SYMBOL(__sk_mem_schedule); 2569 2570 /** 2571 * __sk_mem_reduce_allocated - reclaim memory_allocated 2572 * @sk: socket 2573 * @amount: number of quanta 2574 * 2575 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 2576 */ 2577 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 2578 { 2579 sk_memory_allocated_sub(sk, amount); 2580 2581 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 2582 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 2583 2584 if (sk_under_memory_pressure(sk) && 2585 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 2586 sk_leave_memory_pressure(sk); 2587 } 2588 EXPORT_SYMBOL(__sk_mem_reduce_allocated); 2589 2590 /** 2591 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 2592 * @sk: socket 2593 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple) 2594 */ 2595 void __sk_mem_reclaim(struct sock *sk, int amount) 2596 { 2597 amount >>= SK_MEM_QUANTUM_SHIFT; 2598 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT; 2599 __sk_mem_reduce_allocated(sk, amount); 2600 } 2601 EXPORT_SYMBOL(__sk_mem_reclaim); 2602 2603 int sk_set_peek_off(struct sock *sk, int val) 2604 { 2605 sk->sk_peek_off = val; 2606 return 0; 2607 } 2608 EXPORT_SYMBOL_GPL(sk_set_peek_off); 2609 2610 /* 2611 * Set of default routines for initialising struct proto_ops when 2612 * the protocol does not support a particular function. In certain 2613 * cases where it makes no sense for a protocol to have a "do nothing" 2614 * function, some default processing is provided. 2615 */ 2616 2617 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 2618 { 2619 return -EOPNOTSUPP; 2620 } 2621 EXPORT_SYMBOL(sock_no_bind); 2622 2623 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 2624 int len, int flags) 2625 { 2626 return -EOPNOTSUPP; 2627 } 2628 EXPORT_SYMBOL(sock_no_connect); 2629 2630 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 2631 { 2632 return -EOPNOTSUPP; 2633 } 2634 EXPORT_SYMBOL(sock_no_socketpair); 2635 2636 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, 2637 bool kern) 2638 { 2639 return -EOPNOTSUPP; 2640 } 2641 EXPORT_SYMBOL(sock_no_accept); 2642 2643 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 2644 int peer) 2645 { 2646 return -EOPNOTSUPP; 2647 } 2648 EXPORT_SYMBOL(sock_no_getname); 2649 2650 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 2651 { 2652 return -EOPNOTSUPP; 2653 } 2654 EXPORT_SYMBOL(sock_no_ioctl); 2655 2656 int sock_no_listen(struct socket *sock, int backlog) 2657 { 2658 return -EOPNOTSUPP; 2659 } 2660 EXPORT_SYMBOL(sock_no_listen); 2661 2662 int sock_no_shutdown(struct socket *sock, int how) 2663 { 2664 return -EOPNOTSUPP; 2665 } 2666 EXPORT_SYMBOL(sock_no_shutdown); 2667 2668 int sock_no_setsockopt(struct socket *sock, int level, int optname, 2669 char __user *optval, unsigned int optlen) 2670 { 2671 return -EOPNOTSUPP; 2672 } 2673 EXPORT_SYMBOL(sock_no_setsockopt); 2674 2675 int sock_no_getsockopt(struct socket *sock, int level, int optname, 2676 char __user *optval, int __user *optlen) 2677 { 2678 return -EOPNOTSUPP; 2679 } 2680 EXPORT_SYMBOL(sock_no_getsockopt); 2681 2682 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 2683 { 2684 return -EOPNOTSUPP; 2685 } 2686 EXPORT_SYMBOL(sock_no_sendmsg); 2687 2688 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 2689 { 2690 return -EOPNOTSUPP; 2691 } 2692 EXPORT_SYMBOL(sock_no_sendmsg_locked); 2693 2694 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 2695 int flags) 2696 { 2697 return -EOPNOTSUPP; 2698 } 2699 EXPORT_SYMBOL(sock_no_recvmsg); 2700 2701 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 2702 { 2703 /* Mirror missing mmap method error code */ 2704 return -ENODEV; 2705 } 2706 EXPORT_SYMBOL(sock_no_mmap); 2707 2708 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) 2709 { 2710 ssize_t res; 2711 struct msghdr msg = {.msg_flags = flags}; 2712 struct kvec iov; 2713 char *kaddr = kmap(page); 2714 iov.iov_base = kaddr + offset; 2715 iov.iov_len = size; 2716 res = kernel_sendmsg(sock, &msg, &iov, 1, size); 2717 kunmap(page); 2718 return res; 2719 } 2720 EXPORT_SYMBOL(sock_no_sendpage); 2721 2722 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page, 2723 int offset, size_t size, int flags) 2724 { 2725 ssize_t res; 2726 struct msghdr msg = {.msg_flags = flags}; 2727 struct kvec iov; 2728 char *kaddr = kmap(page); 2729 2730 iov.iov_base = kaddr + offset; 2731 iov.iov_len = size; 2732 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size); 2733 kunmap(page); 2734 return res; 2735 } 2736 EXPORT_SYMBOL(sock_no_sendpage_locked); 2737 2738 /* 2739 * Default Socket Callbacks 2740 */ 2741 2742 static void sock_def_wakeup(struct sock *sk) 2743 { 2744 struct socket_wq *wq; 2745 2746 rcu_read_lock(); 2747 wq = rcu_dereference(sk->sk_wq); 2748 if (skwq_has_sleeper(wq)) 2749 wake_up_interruptible_all(&wq->wait); 2750 rcu_read_unlock(); 2751 } 2752 2753 static void sock_def_error_report(struct sock *sk) 2754 { 2755 struct socket_wq *wq; 2756 2757 rcu_read_lock(); 2758 wq = rcu_dereference(sk->sk_wq); 2759 if (skwq_has_sleeper(wq)) 2760 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 2761 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 2762 rcu_read_unlock(); 2763 } 2764 2765 static void sock_def_readable(struct sock *sk) 2766 { 2767 struct socket_wq *wq; 2768 2769 rcu_read_lock(); 2770 wq = rcu_dereference(sk->sk_wq); 2771 if (skwq_has_sleeper(wq)) 2772 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 2773 EPOLLRDNORM | EPOLLRDBAND); 2774 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 2775 rcu_read_unlock(); 2776 } 2777 2778 static void sock_def_write_space(struct sock *sk) 2779 { 2780 struct socket_wq *wq; 2781 2782 rcu_read_lock(); 2783 2784 /* Do not wake up a writer until he can make "significant" 2785 * progress. --DaveM 2786 */ 2787 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) { 2788 wq = rcu_dereference(sk->sk_wq); 2789 if (skwq_has_sleeper(wq)) 2790 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 2791 EPOLLWRNORM | EPOLLWRBAND); 2792 2793 /* Should agree with poll, otherwise some programs break */ 2794 if (sock_writeable(sk)) 2795 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 2796 } 2797 2798 rcu_read_unlock(); 2799 } 2800 2801 static void sock_def_destruct(struct sock *sk) 2802 { 2803 } 2804 2805 void sk_send_sigurg(struct sock *sk) 2806 { 2807 if (sk->sk_socket && sk->sk_socket->file) 2808 if (send_sigurg(&sk->sk_socket->file->f_owner)) 2809 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 2810 } 2811 EXPORT_SYMBOL(sk_send_sigurg); 2812 2813 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 2814 unsigned long expires) 2815 { 2816 if (!mod_timer(timer, expires)) 2817 sock_hold(sk); 2818 } 2819 EXPORT_SYMBOL(sk_reset_timer); 2820 2821 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 2822 { 2823 if (del_timer(timer)) 2824 __sock_put(sk); 2825 } 2826 EXPORT_SYMBOL(sk_stop_timer); 2827 2828 void sock_init_data(struct socket *sock, struct sock *sk) 2829 { 2830 sk_init_common(sk); 2831 sk->sk_send_head = NULL; 2832 2833 timer_setup(&sk->sk_timer, NULL, 0); 2834 2835 sk->sk_allocation = GFP_KERNEL; 2836 sk->sk_rcvbuf = sysctl_rmem_default; 2837 sk->sk_sndbuf = sysctl_wmem_default; 2838 sk->sk_state = TCP_CLOSE; 2839 sk_set_socket(sk, sock); 2840 2841 sock_set_flag(sk, SOCK_ZAPPED); 2842 2843 if (sock) { 2844 sk->sk_type = sock->type; 2845 RCU_INIT_POINTER(sk->sk_wq, sock->wq); 2846 sock->sk = sk; 2847 sk->sk_uid = SOCK_INODE(sock)->i_uid; 2848 } else { 2849 RCU_INIT_POINTER(sk->sk_wq, NULL); 2850 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0); 2851 } 2852 2853 rwlock_init(&sk->sk_callback_lock); 2854 if (sk->sk_kern_sock) 2855 lockdep_set_class_and_name( 2856 &sk->sk_callback_lock, 2857 af_kern_callback_keys + sk->sk_family, 2858 af_family_kern_clock_key_strings[sk->sk_family]); 2859 else 2860 lockdep_set_class_and_name( 2861 &sk->sk_callback_lock, 2862 af_callback_keys + sk->sk_family, 2863 af_family_clock_key_strings[sk->sk_family]); 2864 2865 sk->sk_state_change = sock_def_wakeup; 2866 sk->sk_data_ready = sock_def_readable; 2867 sk->sk_write_space = sock_def_write_space; 2868 sk->sk_error_report = sock_def_error_report; 2869 sk->sk_destruct = sock_def_destruct; 2870 2871 sk->sk_frag.page = NULL; 2872 sk->sk_frag.offset = 0; 2873 sk->sk_peek_off = -1; 2874 2875 sk->sk_peer_pid = NULL; 2876 sk->sk_peer_cred = NULL; 2877 sk->sk_write_pending = 0; 2878 sk->sk_rcvlowat = 1; 2879 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 2880 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 2881 2882 sk->sk_stamp = SK_DEFAULT_STAMP; 2883 #if BITS_PER_LONG==32 2884 seqlock_init(&sk->sk_stamp_seq); 2885 #endif 2886 atomic_set(&sk->sk_zckey, 0); 2887 2888 #ifdef CONFIG_NET_RX_BUSY_POLL 2889 sk->sk_napi_id = 0; 2890 sk->sk_ll_usec = sysctl_net_busy_read; 2891 #endif 2892 2893 sk->sk_max_pacing_rate = ~0UL; 2894 sk->sk_pacing_rate = ~0UL; 2895 sk->sk_pacing_shift = 10; 2896 sk->sk_incoming_cpu = -1; 2897 2898 sk_rx_queue_clear(sk); 2899 /* 2900 * Before updating sk_refcnt, we must commit prior changes to memory 2901 * (Documentation/RCU/rculist_nulls.txt for details) 2902 */ 2903 smp_wmb(); 2904 refcount_set(&sk->sk_refcnt, 1); 2905 atomic_set(&sk->sk_drops, 0); 2906 } 2907 EXPORT_SYMBOL(sock_init_data); 2908 2909 void lock_sock_nested(struct sock *sk, int subclass) 2910 { 2911 might_sleep(); 2912 spin_lock_bh(&sk->sk_lock.slock); 2913 if (sk->sk_lock.owned) 2914 __lock_sock(sk); 2915 sk->sk_lock.owned = 1; 2916 spin_unlock(&sk->sk_lock.slock); 2917 /* 2918 * The sk_lock has mutex_lock() semantics here: 2919 */ 2920 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 2921 local_bh_enable(); 2922 } 2923 EXPORT_SYMBOL(lock_sock_nested); 2924 2925 void release_sock(struct sock *sk) 2926 { 2927 spin_lock_bh(&sk->sk_lock.slock); 2928 if (sk->sk_backlog.tail) 2929 __release_sock(sk); 2930 2931 /* Warning : release_cb() might need to release sk ownership, 2932 * ie call sock_release_ownership(sk) before us. 2933 */ 2934 if (sk->sk_prot->release_cb) 2935 sk->sk_prot->release_cb(sk); 2936 2937 sock_release_ownership(sk); 2938 if (waitqueue_active(&sk->sk_lock.wq)) 2939 wake_up(&sk->sk_lock.wq); 2940 spin_unlock_bh(&sk->sk_lock.slock); 2941 } 2942 EXPORT_SYMBOL(release_sock); 2943 2944 /** 2945 * lock_sock_fast - fast version of lock_sock 2946 * @sk: socket 2947 * 2948 * This version should be used for very small section, where process wont block 2949 * return false if fast path is taken: 2950 * 2951 * sk_lock.slock locked, owned = 0, BH disabled 2952 * 2953 * return true if slow path is taken: 2954 * 2955 * sk_lock.slock unlocked, owned = 1, BH enabled 2956 */ 2957 bool lock_sock_fast(struct sock *sk) 2958 { 2959 might_sleep(); 2960 spin_lock_bh(&sk->sk_lock.slock); 2961 2962 if (!sk->sk_lock.owned) 2963 /* 2964 * Note : We must disable BH 2965 */ 2966 return false; 2967 2968 __lock_sock(sk); 2969 sk->sk_lock.owned = 1; 2970 spin_unlock(&sk->sk_lock.slock); 2971 /* 2972 * The sk_lock has mutex_lock() semantics here: 2973 */ 2974 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_); 2975 local_bh_enable(); 2976 return true; 2977 } 2978 EXPORT_SYMBOL(lock_sock_fast); 2979 2980 int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp) 2981 { 2982 struct timeval tv; 2983 2984 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 2985 tv = ktime_to_timeval(sock_read_timestamp(sk)); 2986 if (tv.tv_sec == -1) 2987 return -ENOENT; 2988 if (tv.tv_sec == 0) { 2989 ktime_t kt = ktime_get_real(); 2990 sock_write_timestamp(sk, kt); 2991 tv = ktime_to_timeval(kt); 2992 } 2993 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0; 2994 } 2995 EXPORT_SYMBOL(sock_get_timestamp); 2996 2997 int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp) 2998 { 2999 struct timespec ts; 3000 3001 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3002 ts = ktime_to_timespec(sock_read_timestamp(sk)); 3003 if (ts.tv_sec == -1) 3004 return -ENOENT; 3005 if (ts.tv_sec == 0) { 3006 ktime_t kt = ktime_get_real(); 3007 sock_write_timestamp(sk, kt); 3008 ts = ktime_to_timespec(sk->sk_stamp); 3009 } 3010 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0; 3011 } 3012 EXPORT_SYMBOL(sock_get_timestampns); 3013 3014 void sock_enable_timestamp(struct sock *sk, int flag) 3015 { 3016 if (!sock_flag(sk, flag)) { 3017 unsigned long previous_flags = sk->sk_flags; 3018 3019 sock_set_flag(sk, flag); 3020 /* 3021 * we just set one of the two flags which require net 3022 * time stamping, but time stamping might have been on 3023 * already because of the other one 3024 */ 3025 if (sock_needs_netstamp(sk) && 3026 !(previous_flags & SK_FLAGS_TIMESTAMP)) 3027 net_enable_timestamp(); 3028 } 3029 } 3030 3031 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 3032 int level, int type) 3033 { 3034 struct sock_exterr_skb *serr; 3035 struct sk_buff *skb; 3036 int copied, err; 3037 3038 err = -EAGAIN; 3039 skb = sock_dequeue_err_skb(sk); 3040 if (skb == NULL) 3041 goto out; 3042 3043 copied = skb->len; 3044 if (copied > len) { 3045 msg->msg_flags |= MSG_TRUNC; 3046 copied = len; 3047 } 3048 err = skb_copy_datagram_msg(skb, 0, msg, copied); 3049 if (err) 3050 goto out_free_skb; 3051 3052 sock_recv_timestamp(msg, sk, skb); 3053 3054 serr = SKB_EXT_ERR(skb); 3055 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 3056 3057 msg->msg_flags |= MSG_ERRQUEUE; 3058 err = copied; 3059 3060 out_free_skb: 3061 kfree_skb(skb); 3062 out: 3063 return err; 3064 } 3065 EXPORT_SYMBOL(sock_recv_errqueue); 3066 3067 /* 3068 * Get a socket option on an socket. 3069 * 3070 * FIX: POSIX 1003.1g is very ambiguous here. It states that 3071 * asynchronous errors should be reported by getsockopt. We assume 3072 * this means if you specify SO_ERROR (otherwise whats the point of it). 3073 */ 3074 int sock_common_getsockopt(struct socket *sock, int level, int optname, 3075 char __user *optval, int __user *optlen) 3076 { 3077 struct sock *sk = sock->sk; 3078 3079 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 3080 } 3081 EXPORT_SYMBOL(sock_common_getsockopt); 3082 3083 #ifdef CONFIG_COMPAT 3084 int compat_sock_common_getsockopt(struct socket *sock, int level, int optname, 3085 char __user *optval, int __user *optlen) 3086 { 3087 struct sock *sk = sock->sk; 3088 3089 if (sk->sk_prot->compat_getsockopt != NULL) 3090 return sk->sk_prot->compat_getsockopt(sk, level, optname, 3091 optval, optlen); 3092 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); 3093 } 3094 EXPORT_SYMBOL(compat_sock_common_getsockopt); 3095 #endif 3096 3097 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 3098 int flags) 3099 { 3100 struct sock *sk = sock->sk; 3101 int addr_len = 0; 3102 int err; 3103 3104 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, 3105 flags & ~MSG_DONTWAIT, &addr_len); 3106 if (err >= 0) 3107 msg->msg_namelen = addr_len; 3108 return err; 3109 } 3110 EXPORT_SYMBOL(sock_common_recvmsg); 3111 3112 /* 3113 * Set socket options on an inet socket. 3114 */ 3115 int sock_common_setsockopt(struct socket *sock, int level, int optname, 3116 char __user *optval, unsigned int optlen) 3117 { 3118 struct sock *sk = sock->sk; 3119 3120 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3121 } 3122 EXPORT_SYMBOL(sock_common_setsockopt); 3123 3124 #ifdef CONFIG_COMPAT 3125 int compat_sock_common_setsockopt(struct socket *sock, int level, int optname, 3126 char __user *optval, unsigned int optlen) 3127 { 3128 struct sock *sk = sock->sk; 3129 3130 if (sk->sk_prot->compat_setsockopt != NULL) 3131 return sk->sk_prot->compat_setsockopt(sk, level, optname, 3132 optval, optlen); 3133 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); 3134 } 3135 EXPORT_SYMBOL(compat_sock_common_setsockopt); 3136 #endif 3137 3138 void sk_common_release(struct sock *sk) 3139 { 3140 if (sk->sk_prot->destroy) 3141 sk->sk_prot->destroy(sk); 3142 3143 /* 3144 * Observation: when sock_common_release is called, processes have 3145 * no access to socket. But net still has. 3146 * Step one, detach it from networking: 3147 * 3148 * A. Remove from hash tables. 3149 */ 3150 3151 sk->sk_prot->unhash(sk); 3152 3153 /* 3154 * In this point socket cannot receive new packets, but it is possible 3155 * that some packets are in flight because some CPU runs receiver and 3156 * did hash table lookup before we unhashed socket. They will achieve 3157 * receive queue and will be purged by socket destructor. 3158 * 3159 * Also we still have packets pending on receive queue and probably, 3160 * our own packets waiting in device queues. sock_destroy will drain 3161 * receive queue, but transmitted packets will delay socket destruction 3162 * until the last reference will be released. 3163 */ 3164 3165 sock_orphan(sk); 3166 3167 xfrm_sk_free_policy(sk); 3168 3169 sk_refcnt_debug_release(sk); 3170 3171 sock_put(sk); 3172 } 3173 EXPORT_SYMBOL(sk_common_release); 3174 3175 void sk_get_meminfo(const struct sock *sk, u32 *mem) 3176 { 3177 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 3178 3179 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 3180 mem[SK_MEMINFO_RCVBUF] = sk->sk_rcvbuf; 3181 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 3182 mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf; 3183 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc; 3184 mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued; 3185 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 3186 mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len; 3187 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops); 3188 } 3189 3190 #ifdef CONFIG_PROC_FS 3191 #define PROTO_INUSE_NR 64 /* should be enough for the first time */ 3192 struct prot_inuse { 3193 int val[PROTO_INUSE_NR]; 3194 }; 3195 3196 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 3197 3198 void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) 3199 { 3200 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); 3201 } 3202 EXPORT_SYMBOL_GPL(sock_prot_inuse_add); 3203 3204 int sock_prot_inuse_get(struct net *net, struct proto *prot) 3205 { 3206 int cpu, idx = prot->inuse_idx; 3207 int res = 0; 3208 3209 for_each_possible_cpu(cpu) 3210 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 3211 3212 return res >= 0 ? res : 0; 3213 } 3214 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 3215 3216 static void sock_inuse_add(struct net *net, int val) 3217 { 3218 this_cpu_add(*net->core.sock_inuse, val); 3219 } 3220 3221 int sock_inuse_get(struct net *net) 3222 { 3223 int cpu, res = 0; 3224 3225 for_each_possible_cpu(cpu) 3226 res += *per_cpu_ptr(net->core.sock_inuse, cpu); 3227 3228 return res; 3229 } 3230 3231 EXPORT_SYMBOL_GPL(sock_inuse_get); 3232 3233 static int __net_init sock_inuse_init_net(struct net *net) 3234 { 3235 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 3236 if (net->core.prot_inuse == NULL) 3237 return -ENOMEM; 3238 3239 net->core.sock_inuse = alloc_percpu(int); 3240 if (net->core.sock_inuse == NULL) 3241 goto out; 3242 3243 return 0; 3244 3245 out: 3246 free_percpu(net->core.prot_inuse); 3247 return -ENOMEM; 3248 } 3249 3250 static void __net_exit sock_inuse_exit_net(struct net *net) 3251 { 3252 free_percpu(net->core.prot_inuse); 3253 free_percpu(net->core.sock_inuse); 3254 } 3255 3256 static struct pernet_operations net_inuse_ops = { 3257 .init = sock_inuse_init_net, 3258 .exit = sock_inuse_exit_net, 3259 }; 3260 3261 static __init int net_inuse_init(void) 3262 { 3263 if (register_pernet_subsys(&net_inuse_ops)) 3264 panic("Cannot initialize net inuse counters"); 3265 3266 return 0; 3267 } 3268 3269 core_initcall(net_inuse_init); 3270 3271 static void assign_proto_idx(struct proto *prot) 3272 { 3273 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 3274 3275 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 3276 pr_err("PROTO_INUSE_NR exhausted\n"); 3277 return; 3278 } 3279 3280 set_bit(prot->inuse_idx, proto_inuse_idx); 3281 } 3282 3283 static void release_proto_idx(struct proto *prot) 3284 { 3285 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 3286 clear_bit(prot->inuse_idx, proto_inuse_idx); 3287 } 3288 #else 3289 static inline void assign_proto_idx(struct proto *prot) 3290 { 3291 } 3292 3293 static inline void release_proto_idx(struct proto *prot) 3294 { 3295 } 3296 3297 static void sock_inuse_add(struct net *net, int val) 3298 { 3299 } 3300 #endif 3301 3302 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 3303 { 3304 if (!rsk_prot) 3305 return; 3306 kfree(rsk_prot->slab_name); 3307 rsk_prot->slab_name = NULL; 3308 kmem_cache_destroy(rsk_prot->slab); 3309 rsk_prot->slab = NULL; 3310 } 3311 3312 static int req_prot_init(const struct proto *prot) 3313 { 3314 struct request_sock_ops *rsk_prot = prot->rsk_prot; 3315 3316 if (!rsk_prot) 3317 return 0; 3318 3319 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 3320 prot->name); 3321 if (!rsk_prot->slab_name) 3322 return -ENOMEM; 3323 3324 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 3325 rsk_prot->obj_size, 0, 3326 SLAB_ACCOUNT | prot->slab_flags, 3327 NULL); 3328 3329 if (!rsk_prot->slab) { 3330 pr_crit("%s: Can't create request sock SLAB cache!\n", 3331 prot->name); 3332 return -ENOMEM; 3333 } 3334 return 0; 3335 } 3336 3337 int proto_register(struct proto *prot, int alloc_slab) 3338 { 3339 if (alloc_slab) { 3340 prot->slab = kmem_cache_create_usercopy(prot->name, 3341 prot->obj_size, 0, 3342 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 3343 prot->slab_flags, 3344 prot->useroffset, prot->usersize, 3345 NULL); 3346 3347 if (prot->slab == NULL) { 3348 pr_crit("%s: Can't create sock SLAB cache!\n", 3349 prot->name); 3350 goto out; 3351 } 3352 3353 if (req_prot_init(prot)) 3354 goto out_free_request_sock_slab; 3355 3356 if (prot->twsk_prot != NULL) { 3357 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name); 3358 3359 if (prot->twsk_prot->twsk_slab_name == NULL) 3360 goto out_free_request_sock_slab; 3361 3362 prot->twsk_prot->twsk_slab = 3363 kmem_cache_create(prot->twsk_prot->twsk_slab_name, 3364 prot->twsk_prot->twsk_obj_size, 3365 0, 3366 SLAB_ACCOUNT | 3367 prot->slab_flags, 3368 NULL); 3369 if (prot->twsk_prot->twsk_slab == NULL) 3370 goto out_free_timewait_sock_slab_name; 3371 } 3372 } 3373 3374 mutex_lock(&proto_list_mutex); 3375 list_add(&prot->node, &proto_list); 3376 assign_proto_idx(prot); 3377 mutex_unlock(&proto_list_mutex); 3378 return 0; 3379 3380 out_free_timewait_sock_slab_name: 3381 kfree(prot->twsk_prot->twsk_slab_name); 3382 out_free_request_sock_slab: 3383 req_prot_cleanup(prot->rsk_prot); 3384 3385 kmem_cache_destroy(prot->slab); 3386 prot->slab = NULL; 3387 out: 3388 return -ENOBUFS; 3389 } 3390 EXPORT_SYMBOL(proto_register); 3391 3392 void proto_unregister(struct proto *prot) 3393 { 3394 mutex_lock(&proto_list_mutex); 3395 release_proto_idx(prot); 3396 list_del(&prot->node); 3397 mutex_unlock(&proto_list_mutex); 3398 3399 kmem_cache_destroy(prot->slab); 3400 prot->slab = NULL; 3401 3402 req_prot_cleanup(prot->rsk_prot); 3403 3404 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) { 3405 kmem_cache_destroy(prot->twsk_prot->twsk_slab); 3406 kfree(prot->twsk_prot->twsk_slab_name); 3407 prot->twsk_prot->twsk_slab = NULL; 3408 } 3409 } 3410 EXPORT_SYMBOL(proto_unregister); 3411 3412 int sock_load_diag_module(int family, int protocol) 3413 { 3414 if (!protocol) { 3415 if (!sock_is_registered(family)) 3416 return -ENOENT; 3417 3418 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 3419 NETLINK_SOCK_DIAG, family); 3420 } 3421 3422 #ifdef CONFIG_INET 3423 if (family == AF_INET && 3424 protocol != IPPROTO_RAW && 3425 !rcu_access_pointer(inet_protos[protocol])) 3426 return -ENOENT; 3427 #endif 3428 3429 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 3430 NETLINK_SOCK_DIAG, family, protocol); 3431 } 3432 EXPORT_SYMBOL(sock_load_diag_module); 3433 3434 #ifdef CONFIG_PROC_FS 3435 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 3436 __acquires(proto_list_mutex) 3437 { 3438 mutex_lock(&proto_list_mutex); 3439 return seq_list_start_head(&proto_list, *pos); 3440 } 3441 3442 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3443 { 3444 return seq_list_next(v, &proto_list, pos); 3445 } 3446 3447 static void proto_seq_stop(struct seq_file *seq, void *v) 3448 __releases(proto_list_mutex) 3449 { 3450 mutex_unlock(&proto_list_mutex); 3451 } 3452 3453 static char proto_method_implemented(const void *method) 3454 { 3455 return method == NULL ? 'n' : 'y'; 3456 } 3457 static long sock_prot_memory_allocated(struct proto *proto) 3458 { 3459 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 3460 } 3461 3462 static char *sock_prot_memory_pressure(struct proto *proto) 3463 { 3464 return proto->memory_pressure != NULL ? 3465 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 3466 } 3467 3468 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 3469 { 3470 3471 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 3472 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 3473 proto->name, 3474 proto->obj_size, 3475 sock_prot_inuse_get(seq_file_net(seq), proto), 3476 sock_prot_memory_allocated(proto), 3477 sock_prot_memory_pressure(proto), 3478 proto->max_header, 3479 proto->slab == NULL ? "no" : "yes", 3480 module_name(proto->owner), 3481 proto_method_implemented(proto->close), 3482 proto_method_implemented(proto->connect), 3483 proto_method_implemented(proto->disconnect), 3484 proto_method_implemented(proto->accept), 3485 proto_method_implemented(proto->ioctl), 3486 proto_method_implemented(proto->init), 3487 proto_method_implemented(proto->destroy), 3488 proto_method_implemented(proto->shutdown), 3489 proto_method_implemented(proto->setsockopt), 3490 proto_method_implemented(proto->getsockopt), 3491 proto_method_implemented(proto->sendmsg), 3492 proto_method_implemented(proto->recvmsg), 3493 proto_method_implemented(proto->sendpage), 3494 proto_method_implemented(proto->bind), 3495 proto_method_implemented(proto->backlog_rcv), 3496 proto_method_implemented(proto->hash), 3497 proto_method_implemented(proto->unhash), 3498 proto_method_implemented(proto->get_port), 3499 proto_method_implemented(proto->enter_memory_pressure)); 3500 } 3501 3502 static int proto_seq_show(struct seq_file *seq, void *v) 3503 { 3504 if (v == &proto_list) 3505 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 3506 "protocol", 3507 "size", 3508 "sockets", 3509 "memory", 3510 "press", 3511 "maxhdr", 3512 "slab", 3513 "module", 3514 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n"); 3515 else 3516 proto_seq_printf(seq, list_entry(v, struct proto, node)); 3517 return 0; 3518 } 3519 3520 static const struct seq_operations proto_seq_ops = { 3521 .start = proto_seq_start, 3522 .next = proto_seq_next, 3523 .stop = proto_seq_stop, 3524 .show = proto_seq_show, 3525 }; 3526 3527 static __net_init int proto_init_net(struct net *net) 3528 { 3529 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 3530 sizeof(struct seq_net_private))) 3531 return -ENOMEM; 3532 3533 return 0; 3534 } 3535 3536 static __net_exit void proto_exit_net(struct net *net) 3537 { 3538 remove_proc_entry("protocols", net->proc_net); 3539 } 3540 3541 3542 static __net_initdata struct pernet_operations proto_net_ops = { 3543 .init = proto_init_net, 3544 .exit = proto_exit_net, 3545 }; 3546 3547 static int __init proto_init(void) 3548 { 3549 return register_pernet_subsys(&proto_net_ops); 3550 } 3551 3552 subsys_initcall(proto_init); 3553 3554 #endif /* PROC_FS */ 3555 3556 #ifdef CONFIG_NET_RX_BUSY_POLL 3557 bool sk_busy_loop_end(void *p, unsigned long start_time) 3558 { 3559 struct sock *sk = p; 3560 3561 return !skb_queue_empty(&sk->sk_receive_queue) || 3562 sk_busy_loop_timeout(sk, start_time); 3563 } 3564 EXPORT_SYMBOL(sk_busy_loop_end); 3565 #endif /* CONFIG_NET_RX_BUSY_POLL */ 3566