1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Generic socket support routines. Memory allocators, socket lock/release 8 * handler for protocols to use and generic option handler. 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Florian La Roche, <flla@stud.uni-sb.de> 13 * Alan Cox, <A.Cox@swansea.ac.uk> 14 * 15 * Fixes: 16 * Alan Cox : Numerous verify_area() problems 17 * Alan Cox : Connecting on a connecting socket 18 * now returns an error for tcp. 19 * Alan Cox : sock->protocol is set correctly. 20 * and is not sometimes left as 0. 21 * Alan Cox : connect handles icmp errors on a 22 * connect properly. Unfortunately there 23 * is a restart syscall nasty there. I 24 * can't match BSD without hacking the C 25 * library. Ideas urgently sought! 26 * Alan Cox : Disallow bind() to addresses that are 27 * not ours - especially broadcast ones!! 28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost) 29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets, 30 * instead they leave that for the DESTROY timer. 31 * Alan Cox : Clean up error flag in accept 32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer 33 * was buggy. Put a remove_sock() in the handler 34 * for memory when we hit 0. Also altered the timer 35 * code. The ACK stuff can wait and needs major 36 * TCP layer surgery. 37 * Alan Cox : Fixed TCP ack bug, removed remove sock 38 * and fixed timer/inet_bh race. 39 * Alan Cox : Added zapped flag for TCP 40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code 41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb 42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources 43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing. 44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so... 45 * Rick Sladkey : Relaxed UDP rules for matching packets. 46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support 47 * Pauline Middelink : identd support 48 * Alan Cox : Fixed connect() taking signals I think. 49 * Alan Cox : SO_LINGER supported 50 * Alan Cox : Error reporting fixes 51 * Anonymous : inet_create tidied up (sk->reuse setting) 52 * Alan Cox : inet sockets don't set sk->type! 53 * Alan Cox : Split socket option code 54 * Alan Cox : Callbacks 55 * Alan Cox : Nagle flag for Charles & Johannes stuff 56 * Alex : Removed restriction on inet fioctl 57 * Alan Cox : Splitting INET from NET core 58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt() 59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code 60 * Alan Cox : Split IP from generic code 61 * Alan Cox : New kfree_skbmem() 62 * Alan Cox : Make SO_DEBUG superuser only. 63 * Alan Cox : Allow anyone to clear SO_DEBUG 64 * (compatibility fix) 65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput. 66 * Alan Cox : Allocator for a socket is settable. 67 * Alan Cox : SO_ERROR includes soft errors. 68 * Alan Cox : Allow NULL arguments on some SO_ opts 69 * Alan Cox : Generic socket allocation to make hooks 70 * easier (suggested by Craig Metz). 71 * Michael Pall : SO_ERROR returns positive errno again 72 * Steve Whitehouse: Added default destructor to free 73 * protocol private data. 74 * Steve Whitehouse: Added various other default routines 75 * common to several socket families. 76 * Chris Evans : Call suser() check last on F_SETOWN 77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER. 78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s() 79 * Andi Kleen : Fix write_space callback 80 * Chris Evans : Security fixes - signedness again 81 * Arnaldo C. Melo : cleanups, use skb_queue_purge 82 * 83 * To Fix: 84 */ 85 86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 87 88 #include <asm/unaligned.h> 89 #include <linux/capability.h> 90 #include <linux/errno.h> 91 #include <linux/errqueue.h> 92 #include <linux/types.h> 93 #include <linux/socket.h> 94 #include <linux/in.h> 95 #include <linux/kernel.h> 96 #include <linux/module.h> 97 #include <linux/proc_fs.h> 98 #include <linux/seq_file.h> 99 #include <linux/sched.h> 100 #include <linux/sched/mm.h> 101 #include <linux/timer.h> 102 #include <linux/string.h> 103 #include <linux/sockios.h> 104 #include <linux/net.h> 105 #include <linux/mm.h> 106 #include <linux/slab.h> 107 #include <linux/interrupt.h> 108 #include <linux/poll.h> 109 #include <linux/tcp.h> 110 #include <linux/udp.h> 111 #include <linux/init.h> 112 #include <linux/highmem.h> 113 #include <linux/user_namespace.h> 114 #include <linux/static_key.h> 115 #include <linux/memcontrol.h> 116 #include <linux/prefetch.h> 117 #include <linux/compat.h> 118 #include <linux/mroute.h> 119 #include <linux/mroute6.h> 120 #include <linux/icmpv6.h> 121 122 #include <linux/uaccess.h> 123 124 #include <linux/netdevice.h> 125 #include <net/protocol.h> 126 #include <linux/skbuff.h> 127 #include <net/net_namespace.h> 128 #include <net/request_sock.h> 129 #include <net/sock.h> 130 #include <linux/net_tstamp.h> 131 #include <net/xfrm.h> 132 #include <linux/ipsec.h> 133 #include <net/cls_cgroup.h> 134 #include <net/netprio_cgroup.h> 135 #include <linux/sock_diag.h> 136 137 #include <linux/filter.h> 138 #include <net/sock_reuseport.h> 139 #include <net/bpf_sk_storage.h> 140 141 #include <trace/events/sock.h> 142 143 #include <net/tcp.h> 144 #include <net/busy_poll.h> 145 #include <net/phonet/phonet.h> 146 147 #include <linux/ethtool.h> 148 149 #include "dev.h" 150 151 static DEFINE_MUTEX(proto_list_mutex); 152 static LIST_HEAD(proto_list); 153 154 static void sock_def_write_space_wfree(struct sock *sk); 155 static void sock_def_write_space(struct sock *sk); 156 157 /** 158 * sk_ns_capable - General socket capability test 159 * @sk: Socket to use a capability on or through 160 * @user_ns: The user namespace of the capability to use 161 * @cap: The capability to use 162 * 163 * Test to see if the opener of the socket had when the socket was 164 * created and the current process has the capability @cap in the user 165 * namespace @user_ns. 166 */ 167 bool sk_ns_capable(const struct sock *sk, 168 struct user_namespace *user_ns, int cap) 169 { 170 return file_ns_capable(sk->sk_socket->file, user_ns, cap) && 171 ns_capable(user_ns, cap); 172 } 173 EXPORT_SYMBOL(sk_ns_capable); 174 175 /** 176 * sk_capable - Socket global capability test 177 * @sk: Socket to use a capability on or through 178 * @cap: The global capability to use 179 * 180 * Test to see if the opener of the socket had when the socket was 181 * created and the current process has the capability @cap in all user 182 * namespaces. 183 */ 184 bool sk_capable(const struct sock *sk, int cap) 185 { 186 return sk_ns_capable(sk, &init_user_ns, cap); 187 } 188 EXPORT_SYMBOL(sk_capable); 189 190 /** 191 * sk_net_capable - Network namespace socket capability test 192 * @sk: Socket to use a capability on or through 193 * @cap: The capability to use 194 * 195 * Test to see if the opener of the socket had when the socket was created 196 * and the current process has the capability @cap over the network namespace 197 * the socket is a member of. 198 */ 199 bool sk_net_capable(const struct sock *sk, int cap) 200 { 201 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap); 202 } 203 EXPORT_SYMBOL(sk_net_capable); 204 205 /* 206 * Each address family might have different locking rules, so we have 207 * one slock key per address family and separate keys for internal and 208 * userspace sockets. 209 */ 210 static struct lock_class_key af_family_keys[AF_MAX]; 211 static struct lock_class_key af_family_kern_keys[AF_MAX]; 212 static struct lock_class_key af_family_slock_keys[AF_MAX]; 213 static struct lock_class_key af_family_kern_slock_keys[AF_MAX]; 214 215 /* 216 * Make lock validator output more readable. (we pre-construct these 217 * strings build-time, so that runtime initialization of socket 218 * locks is fast): 219 */ 220 221 #define _sock_locks(x) \ 222 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \ 223 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \ 224 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \ 225 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \ 226 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \ 227 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \ 228 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \ 229 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \ 230 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \ 231 x "27" , x "28" , x "AF_CAN" , \ 232 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \ 233 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \ 234 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \ 235 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \ 236 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \ 237 x "AF_MCTP" , \ 238 x "AF_MAX" 239 240 static const char *const af_family_key_strings[AF_MAX+1] = { 241 _sock_locks("sk_lock-") 242 }; 243 static const char *const af_family_slock_key_strings[AF_MAX+1] = { 244 _sock_locks("slock-") 245 }; 246 static const char *const af_family_clock_key_strings[AF_MAX+1] = { 247 _sock_locks("clock-") 248 }; 249 250 static const char *const af_family_kern_key_strings[AF_MAX+1] = { 251 _sock_locks("k-sk_lock-") 252 }; 253 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = { 254 _sock_locks("k-slock-") 255 }; 256 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = { 257 _sock_locks("k-clock-") 258 }; 259 static const char *const af_family_rlock_key_strings[AF_MAX+1] = { 260 _sock_locks("rlock-") 261 }; 262 static const char *const af_family_wlock_key_strings[AF_MAX+1] = { 263 _sock_locks("wlock-") 264 }; 265 static const char *const af_family_elock_key_strings[AF_MAX+1] = { 266 _sock_locks("elock-") 267 }; 268 269 /* 270 * sk_callback_lock and sk queues locking rules are per-address-family, 271 * so split the lock classes by using a per-AF key: 272 */ 273 static struct lock_class_key af_callback_keys[AF_MAX]; 274 static struct lock_class_key af_rlock_keys[AF_MAX]; 275 static struct lock_class_key af_wlock_keys[AF_MAX]; 276 static struct lock_class_key af_elock_keys[AF_MAX]; 277 static struct lock_class_key af_kern_callback_keys[AF_MAX]; 278 279 /* Run time adjustable parameters. */ 280 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX; 281 EXPORT_SYMBOL(sysctl_wmem_max); 282 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; 283 EXPORT_SYMBOL(sysctl_rmem_max); 284 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; 285 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; 286 int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE; 287 288 /* Maximal space eaten by iovec or ancillary data plus some space */ 289 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512); 290 EXPORT_SYMBOL(sysctl_optmem_max); 291 292 int sysctl_tstamp_allow_data __read_mostly = 1; 293 294 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key); 295 EXPORT_SYMBOL_GPL(memalloc_socks_key); 296 297 /** 298 * sk_set_memalloc - sets %SOCK_MEMALLOC 299 * @sk: socket to set it on 300 * 301 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves. 302 * It's the responsibility of the admin to adjust min_free_kbytes 303 * to meet the requirements 304 */ 305 void sk_set_memalloc(struct sock *sk) 306 { 307 sock_set_flag(sk, SOCK_MEMALLOC); 308 sk->sk_allocation |= __GFP_MEMALLOC; 309 static_branch_inc(&memalloc_socks_key); 310 } 311 EXPORT_SYMBOL_GPL(sk_set_memalloc); 312 313 void sk_clear_memalloc(struct sock *sk) 314 { 315 sock_reset_flag(sk, SOCK_MEMALLOC); 316 sk->sk_allocation &= ~__GFP_MEMALLOC; 317 static_branch_dec(&memalloc_socks_key); 318 319 /* 320 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward 321 * progress of swapping. SOCK_MEMALLOC may be cleared while 322 * it has rmem allocations due to the last swapfile being deactivated 323 * but there is a risk that the socket is unusable due to exceeding 324 * the rmem limits. Reclaim the reserves and obey rmem limits again. 325 */ 326 sk_mem_reclaim(sk); 327 } 328 EXPORT_SYMBOL_GPL(sk_clear_memalloc); 329 330 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) 331 { 332 int ret; 333 unsigned int noreclaim_flag; 334 335 /* these should have been dropped before queueing */ 336 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); 337 338 noreclaim_flag = memalloc_noreclaim_save(); 339 ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv, 340 tcp_v6_do_rcv, 341 tcp_v4_do_rcv, 342 sk, skb); 343 memalloc_noreclaim_restore(noreclaim_flag); 344 345 return ret; 346 } 347 EXPORT_SYMBOL(__sk_backlog_rcv); 348 349 void sk_error_report(struct sock *sk) 350 { 351 sk->sk_error_report(sk); 352 353 switch (sk->sk_family) { 354 case AF_INET: 355 fallthrough; 356 case AF_INET6: 357 trace_inet_sk_error_report(sk); 358 break; 359 default: 360 break; 361 } 362 } 363 EXPORT_SYMBOL(sk_error_report); 364 365 int sock_get_timeout(long timeo, void *optval, bool old_timeval) 366 { 367 struct __kernel_sock_timeval tv; 368 369 if (timeo == MAX_SCHEDULE_TIMEOUT) { 370 tv.tv_sec = 0; 371 tv.tv_usec = 0; 372 } else { 373 tv.tv_sec = timeo / HZ; 374 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ; 375 } 376 377 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 378 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec }; 379 *(struct old_timeval32 *)optval = tv32; 380 return sizeof(tv32); 381 } 382 383 if (old_timeval) { 384 struct __kernel_old_timeval old_tv; 385 old_tv.tv_sec = tv.tv_sec; 386 old_tv.tv_usec = tv.tv_usec; 387 *(struct __kernel_old_timeval *)optval = old_tv; 388 return sizeof(old_tv); 389 } 390 391 *(struct __kernel_sock_timeval *)optval = tv; 392 return sizeof(tv); 393 } 394 EXPORT_SYMBOL(sock_get_timeout); 395 396 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv, 397 sockptr_t optval, int optlen, bool old_timeval) 398 { 399 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) { 400 struct old_timeval32 tv32; 401 402 if (optlen < sizeof(tv32)) 403 return -EINVAL; 404 405 if (copy_from_sockptr(&tv32, optval, sizeof(tv32))) 406 return -EFAULT; 407 tv->tv_sec = tv32.tv_sec; 408 tv->tv_usec = tv32.tv_usec; 409 } else if (old_timeval) { 410 struct __kernel_old_timeval old_tv; 411 412 if (optlen < sizeof(old_tv)) 413 return -EINVAL; 414 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv))) 415 return -EFAULT; 416 tv->tv_sec = old_tv.tv_sec; 417 tv->tv_usec = old_tv.tv_usec; 418 } else { 419 if (optlen < sizeof(*tv)) 420 return -EINVAL; 421 if (copy_from_sockptr(tv, optval, sizeof(*tv))) 422 return -EFAULT; 423 } 424 425 return 0; 426 } 427 EXPORT_SYMBOL(sock_copy_user_timeval); 428 429 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen, 430 bool old_timeval) 431 { 432 struct __kernel_sock_timeval tv; 433 int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval); 434 long val; 435 436 if (err) 437 return err; 438 439 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC) 440 return -EDOM; 441 442 if (tv.tv_sec < 0) { 443 static int warned __read_mostly; 444 445 WRITE_ONCE(*timeo_p, 0); 446 if (warned < 10 && net_ratelimit()) { 447 warned++; 448 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n", 449 __func__, current->comm, task_pid_nr(current)); 450 } 451 return 0; 452 } 453 val = MAX_SCHEDULE_TIMEOUT; 454 if ((tv.tv_sec || tv.tv_usec) && 455 (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))) 456 val = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, 457 USEC_PER_SEC / HZ); 458 WRITE_ONCE(*timeo_p, val); 459 return 0; 460 } 461 462 static bool sock_needs_netstamp(const struct sock *sk) 463 { 464 switch (sk->sk_family) { 465 case AF_UNSPEC: 466 case AF_UNIX: 467 return false; 468 default: 469 return true; 470 } 471 } 472 473 static void sock_disable_timestamp(struct sock *sk, unsigned long flags) 474 { 475 if (sk->sk_flags & flags) { 476 sk->sk_flags &= ~flags; 477 if (sock_needs_netstamp(sk) && 478 !(sk->sk_flags & SK_FLAGS_TIMESTAMP)) 479 net_disable_timestamp(); 480 } 481 } 482 483 484 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) 485 { 486 unsigned long flags; 487 struct sk_buff_head *list = &sk->sk_receive_queue; 488 489 if (atomic_read(&sk->sk_rmem_alloc) >= READ_ONCE(sk->sk_rcvbuf)) { 490 atomic_inc(&sk->sk_drops); 491 trace_sock_rcvqueue_full(sk, skb); 492 return -ENOMEM; 493 } 494 495 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 496 atomic_inc(&sk->sk_drops); 497 return -ENOBUFS; 498 } 499 500 skb->dev = NULL; 501 skb_set_owner_r(skb, sk); 502 503 /* we escape from rcu protected region, make sure we dont leak 504 * a norefcounted dst 505 */ 506 skb_dst_force(skb); 507 508 spin_lock_irqsave(&list->lock, flags); 509 sock_skb_set_dropcount(sk, skb); 510 __skb_queue_tail(list, skb); 511 spin_unlock_irqrestore(&list->lock, flags); 512 513 if (!sock_flag(sk, SOCK_DEAD)) 514 sk->sk_data_ready(sk); 515 return 0; 516 } 517 EXPORT_SYMBOL(__sock_queue_rcv_skb); 518 519 int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb, 520 enum skb_drop_reason *reason) 521 { 522 enum skb_drop_reason drop_reason; 523 int err; 524 525 err = sk_filter(sk, skb); 526 if (err) { 527 drop_reason = SKB_DROP_REASON_SOCKET_FILTER; 528 goto out; 529 } 530 err = __sock_queue_rcv_skb(sk, skb); 531 switch (err) { 532 case -ENOMEM: 533 drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF; 534 break; 535 case -ENOBUFS: 536 drop_reason = SKB_DROP_REASON_PROTO_MEM; 537 break; 538 default: 539 drop_reason = SKB_NOT_DROPPED_YET; 540 break; 541 } 542 out: 543 if (reason) 544 *reason = drop_reason; 545 return err; 546 } 547 EXPORT_SYMBOL(sock_queue_rcv_skb_reason); 548 549 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, 550 const int nested, unsigned int trim_cap, bool refcounted) 551 { 552 int rc = NET_RX_SUCCESS; 553 554 if (sk_filter_trim_cap(sk, skb, trim_cap)) 555 goto discard_and_relse; 556 557 skb->dev = NULL; 558 559 if (sk_rcvqueues_full(sk, READ_ONCE(sk->sk_rcvbuf))) { 560 atomic_inc(&sk->sk_drops); 561 goto discard_and_relse; 562 } 563 if (nested) 564 bh_lock_sock_nested(sk); 565 else 566 bh_lock_sock(sk); 567 if (!sock_owned_by_user(sk)) { 568 /* 569 * trylock + unlock semantics: 570 */ 571 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_); 572 573 rc = sk_backlog_rcv(sk, skb); 574 575 mutex_release(&sk->sk_lock.dep_map, _RET_IP_); 576 } else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) { 577 bh_unlock_sock(sk); 578 atomic_inc(&sk->sk_drops); 579 goto discard_and_relse; 580 } 581 582 bh_unlock_sock(sk); 583 out: 584 if (refcounted) 585 sock_put(sk); 586 return rc; 587 discard_and_relse: 588 kfree_skb(skb); 589 goto out; 590 } 591 EXPORT_SYMBOL(__sk_receive_skb); 592 593 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *, 594 u32)); 595 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 596 u32)); 597 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie) 598 { 599 struct dst_entry *dst = __sk_dst_get(sk); 600 601 if (dst && dst->obsolete && 602 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 603 dst, cookie) == NULL) { 604 sk_tx_queue_clear(sk); 605 WRITE_ONCE(sk->sk_dst_pending_confirm, 0); 606 RCU_INIT_POINTER(sk->sk_dst_cache, NULL); 607 dst_release(dst); 608 return NULL; 609 } 610 611 return dst; 612 } 613 EXPORT_SYMBOL(__sk_dst_check); 614 615 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie) 616 { 617 struct dst_entry *dst = sk_dst_get(sk); 618 619 if (dst && dst->obsolete && 620 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check, 621 dst, cookie) == NULL) { 622 sk_dst_reset(sk); 623 dst_release(dst); 624 return NULL; 625 } 626 627 return dst; 628 } 629 EXPORT_SYMBOL(sk_dst_check); 630 631 static int sock_bindtoindex_locked(struct sock *sk, int ifindex) 632 { 633 int ret = -ENOPROTOOPT; 634 #ifdef CONFIG_NETDEVICES 635 struct net *net = sock_net(sk); 636 637 /* Sorry... */ 638 ret = -EPERM; 639 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW)) 640 goto out; 641 642 ret = -EINVAL; 643 if (ifindex < 0) 644 goto out; 645 646 /* Paired with all READ_ONCE() done locklessly. */ 647 WRITE_ONCE(sk->sk_bound_dev_if, ifindex); 648 649 if (sk->sk_prot->rehash) 650 sk->sk_prot->rehash(sk); 651 sk_dst_reset(sk); 652 653 ret = 0; 654 655 out: 656 #endif 657 658 return ret; 659 } 660 661 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk) 662 { 663 int ret; 664 665 if (lock_sk) 666 lock_sock(sk); 667 ret = sock_bindtoindex_locked(sk, ifindex); 668 if (lock_sk) 669 release_sock(sk); 670 671 return ret; 672 } 673 EXPORT_SYMBOL(sock_bindtoindex); 674 675 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) 676 { 677 int ret = -ENOPROTOOPT; 678 #ifdef CONFIG_NETDEVICES 679 struct net *net = sock_net(sk); 680 char devname[IFNAMSIZ]; 681 int index; 682 683 ret = -EINVAL; 684 if (optlen < 0) 685 goto out; 686 687 /* Bind this socket to a particular device like "eth0", 688 * as specified in the passed interface name. If the 689 * name is "" or the option length is zero the socket 690 * is not bound. 691 */ 692 if (optlen > IFNAMSIZ - 1) 693 optlen = IFNAMSIZ - 1; 694 memset(devname, 0, sizeof(devname)); 695 696 ret = -EFAULT; 697 if (copy_from_sockptr(devname, optval, optlen)) 698 goto out; 699 700 index = 0; 701 if (devname[0] != '\0') { 702 struct net_device *dev; 703 704 rcu_read_lock(); 705 dev = dev_get_by_name_rcu(net, devname); 706 if (dev) 707 index = dev->ifindex; 708 rcu_read_unlock(); 709 ret = -ENODEV; 710 if (!dev) 711 goto out; 712 } 713 714 sockopt_lock_sock(sk); 715 ret = sock_bindtoindex_locked(sk, index); 716 sockopt_release_sock(sk); 717 out: 718 #endif 719 720 return ret; 721 } 722 723 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval, 724 sockptr_t optlen, int len) 725 { 726 int ret = -ENOPROTOOPT; 727 #ifdef CONFIG_NETDEVICES 728 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); 729 struct net *net = sock_net(sk); 730 char devname[IFNAMSIZ]; 731 732 if (bound_dev_if == 0) { 733 len = 0; 734 goto zero; 735 } 736 737 ret = -EINVAL; 738 if (len < IFNAMSIZ) 739 goto out; 740 741 ret = netdev_get_name(net, devname, bound_dev_if); 742 if (ret) 743 goto out; 744 745 len = strlen(devname) + 1; 746 747 ret = -EFAULT; 748 if (copy_to_sockptr(optval, devname, len)) 749 goto out; 750 751 zero: 752 ret = -EFAULT; 753 if (copy_to_sockptr(optlen, &len, sizeof(int))) 754 goto out; 755 756 ret = 0; 757 758 out: 759 #endif 760 761 return ret; 762 } 763 764 bool sk_mc_loop(struct sock *sk) 765 { 766 if (dev_recursion_level()) 767 return false; 768 if (!sk) 769 return true; 770 /* IPV6_ADDRFORM can change sk->sk_family under us. */ 771 switch (READ_ONCE(sk->sk_family)) { 772 case AF_INET: 773 return inet_test_bit(MC_LOOP, sk); 774 #if IS_ENABLED(CONFIG_IPV6) 775 case AF_INET6: 776 return inet6_sk(sk)->mc_loop; 777 #endif 778 } 779 WARN_ON_ONCE(1); 780 return true; 781 } 782 EXPORT_SYMBOL(sk_mc_loop); 783 784 void sock_set_reuseaddr(struct sock *sk) 785 { 786 lock_sock(sk); 787 sk->sk_reuse = SK_CAN_REUSE; 788 release_sock(sk); 789 } 790 EXPORT_SYMBOL(sock_set_reuseaddr); 791 792 void sock_set_reuseport(struct sock *sk) 793 { 794 lock_sock(sk); 795 sk->sk_reuseport = true; 796 release_sock(sk); 797 } 798 EXPORT_SYMBOL(sock_set_reuseport); 799 800 void sock_no_linger(struct sock *sk) 801 { 802 lock_sock(sk); 803 WRITE_ONCE(sk->sk_lingertime, 0); 804 sock_set_flag(sk, SOCK_LINGER); 805 release_sock(sk); 806 } 807 EXPORT_SYMBOL(sock_no_linger); 808 809 void sock_set_priority(struct sock *sk, u32 priority) 810 { 811 lock_sock(sk); 812 WRITE_ONCE(sk->sk_priority, priority); 813 release_sock(sk); 814 } 815 EXPORT_SYMBOL(sock_set_priority); 816 817 void sock_set_sndtimeo(struct sock *sk, s64 secs) 818 { 819 lock_sock(sk); 820 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1) 821 WRITE_ONCE(sk->sk_sndtimeo, secs * HZ); 822 else 823 WRITE_ONCE(sk->sk_sndtimeo, MAX_SCHEDULE_TIMEOUT); 824 release_sock(sk); 825 } 826 EXPORT_SYMBOL(sock_set_sndtimeo); 827 828 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns) 829 { 830 if (val) { 831 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new); 832 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, ns); 833 sock_set_flag(sk, SOCK_RCVTSTAMP); 834 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 835 } else { 836 sock_reset_flag(sk, SOCK_RCVTSTAMP); 837 sock_reset_flag(sk, SOCK_RCVTSTAMPNS); 838 } 839 } 840 841 void sock_enable_timestamps(struct sock *sk) 842 { 843 lock_sock(sk); 844 __sock_set_timestamps(sk, true, false, true); 845 release_sock(sk); 846 } 847 EXPORT_SYMBOL(sock_enable_timestamps); 848 849 void sock_set_timestamp(struct sock *sk, int optname, bool valbool) 850 { 851 switch (optname) { 852 case SO_TIMESTAMP_OLD: 853 __sock_set_timestamps(sk, valbool, false, false); 854 break; 855 case SO_TIMESTAMP_NEW: 856 __sock_set_timestamps(sk, valbool, true, false); 857 break; 858 case SO_TIMESTAMPNS_OLD: 859 __sock_set_timestamps(sk, valbool, false, true); 860 break; 861 case SO_TIMESTAMPNS_NEW: 862 __sock_set_timestamps(sk, valbool, true, true); 863 break; 864 } 865 } 866 867 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index) 868 { 869 struct net *net = sock_net(sk); 870 struct net_device *dev = NULL; 871 bool match = false; 872 int *vclock_index; 873 int i, num; 874 875 if (sk->sk_bound_dev_if) 876 dev = dev_get_by_index(net, sk->sk_bound_dev_if); 877 878 if (!dev) { 879 pr_err("%s: sock not bind to device\n", __func__); 880 return -EOPNOTSUPP; 881 } 882 883 num = ethtool_get_phc_vclocks(dev, &vclock_index); 884 dev_put(dev); 885 886 for (i = 0; i < num; i++) { 887 if (*(vclock_index + i) == phc_index) { 888 match = true; 889 break; 890 } 891 } 892 893 if (num > 0) 894 kfree(vclock_index); 895 896 if (!match) 897 return -EINVAL; 898 899 WRITE_ONCE(sk->sk_bind_phc, phc_index); 900 901 return 0; 902 } 903 904 int sock_set_timestamping(struct sock *sk, int optname, 905 struct so_timestamping timestamping) 906 { 907 int val = timestamping.flags; 908 int ret; 909 910 if (val & ~SOF_TIMESTAMPING_MASK) 911 return -EINVAL; 912 913 if (val & SOF_TIMESTAMPING_OPT_ID_TCP && 914 !(val & SOF_TIMESTAMPING_OPT_ID)) 915 return -EINVAL; 916 917 if (val & SOF_TIMESTAMPING_OPT_ID && 918 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { 919 if (sk_is_tcp(sk)) { 920 if ((1 << sk->sk_state) & 921 (TCPF_CLOSE | TCPF_LISTEN)) 922 return -EINVAL; 923 if (val & SOF_TIMESTAMPING_OPT_ID_TCP) 924 atomic_set(&sk->sk_tskey, tcp_sk(sk)->write_seq); 925 else 926 atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una); 927 } else { 928 atomic_set(&sk->sk_tskey, 0); 929 } 930 } 931 932 if (val & SOF_TIMESTAMPING_OPT_STATS && 933 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) 934 return -EINVAL; 935 936 if (val & SOF_TIMESTAMPING_BIND_PHC) { 937 ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc); 938 if (ret) 939 return ret; 940 } 941 942 WRITE_ONCE(sk->sk_tsflags, val); 943 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW); 944 945 if (val & SOF_TIMESTAMPING_RX_SOFTWARE) 946 sock_enable_timestamp(sk, 947 SOCK_TIMESTAMPING_RX_SOFTWARE); 948 else 949 sock_disable_timestamp(sk, 950 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE)); 951 return 0; 952 } 953 954 void sock_set_keepalive(struct sock *sk) 955 { 956 lock_sock(sk); 957 if (sk->sk_prot->keepalive) 958 sk->sk_prot->keepalive(sk, true); 959 sock_valbool_flag(sk, SOCK_KEEPOPEN, true); 960 release_sock(sk); 961 } 962 EXPORT_SYMBOL(sock_set_keepalive); 963 964 static void __sock_set_rcvbuf(struct sock *sk, int val) 965 { 966 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it 967 * as a negative value. 968 */ 969 val = min_t(int, val, INT_MAX / 2); 970 sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 971 972 /* We double it on the way in to account for "struct sk_buff" etc. 973 * overhead. Applications assume that the SO_RCVBUF setting they make 974 * will allow that much actual data to be received on that socket. 975 * 976 * Applications are unaware that "struct sk_buff" and other overheads 977 * allocate from the receive buffer during socket buffer allocation. 978 * 979 * And after considering the possible alternatives, returning the value 980 * we actually used in getsockopt is the most desirable behavior. 981 */ 982 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF)); 983 } 984 985 void sock_set_rcvbuf(struct sock *sk, int val) 986 { 987 lock_sock(sk); 988 __sock_set_rcvbuf(sk, val); 989 release_sock(sk); 990 } 991 EXPORT_SYMBOL(sock_set_rcvbuf); 992 993 static void __sock_set_mark(struct sock *sk, u32 val) 994 { 995 if (val != sk->sk_mark) { 996 WRITE_ONCE(sk->sk_mark, val); 997 sk_dst_reset(sk); 998 } 999 } 1000 1001 void sock_set_mark(struct sock *sk, u32 val) 1002 { 1003 lock_sock(sk); 1004 __sock_set_mark(sk, val); 1005 release_sock(sk); 1006 } 1007 EXPORT_SYMBOL(sock_set_mark); 1008 1009 static void sock_release_reserved_memory(struct sock *sk, int bytes) 1010 { 1011 /* Round down bytes to multiple of pages */ 1012 bytes = round_down(bytes, PAGE_SIZE); 1013 1014 WARN_ON(bytes > sk->sk_reserved_mem); 1015 WRITE_ONCE(sk->sk_reserved_mem, sk->sk_reserved_mem - bytes); 1016 sk_mem_reclaim(sk); 1017 } 1018 1019 static int sock_reserve_memory(struct sock *sk, int bytes) 1020 { 1021 long allocated; 1022 bool charged; 1023 int pages; 1024 1025 if (!mem_cgroup_sockets_enabled || !sk->sk_memcg || !sk_has_account(sk)) 1026 return -EOPNOTSUPP; 1027 1028 if (!bytes) 1029 return 0; 1030 1031 pages = sk_mem_pages(bytes); 1032 1033 /* pre-charge to memcg */ 1034 charged = mem_cgroup_charge_skmem(sk->sk_memcg, pages, 1035 GFP_KERNEL | __GFP_RETRY_MAYFAIL); 1036 if (!charged) 1037 return -ENOMEM; 1038 1039 /* pre-charge to forward_alloc */ 1040 sk_memory_allocated_add(sk, pages); 1041 allocated = sk_memory_allocated(sk); 1042 /* If the system goes into memory pressure with this 1043 * precharge, give up and return error. 1044 */ 1045 if (allocated > sk_prot_mem_limits(sk, 1)) { 1046 sk_memory_allocated_sub(sk, pages); 1047 mem_cgroup_uncharge_skmem(sk->sk_memcg, pages); 1048 return -ENOMEM; 1049 } 1050 sk_forward_alloc_add(sk, pages << PAGE_SHIFT); 1051 1052 WRITE_ONCE(sk->sk_reserved_mem, 1053 sk->sk_reserved_mem + (pages << PAGE_SHIFT)); 1054 1055 return 0; 1056 } 1057 1058 void sockopt_lock_sock(struct sock *sk) 1059 { 1060 /* When current->bpf_ctx is set, the setsockopt is called from 1061 * a bpf prog. bpf has ensured the sk lock has been 1062 * acquired before calling setsockopt(). 1063 */ 1064 if (has_current_bpf_ctx()) 1065 return; 1066 1067 lock_sock(sk); 1068 } 1069 EXPORT_SYMBOL(sockopt_lock_sock); 1070 1071 void sockopt_release_sock(struct sock *sk) 1072 { 1073 if (has_current_bpf_ctx()) 1074 return; 1075 1076 release_sock(sk); 1077 } 1078 EXPORT_SYMBOL(sockopt_release_sock); 1079 1080 bool sockopt_ns_capable(struct user_namespace *ns, int cap) 1081 { 1082 return has_current_bpf_ctx() || ns_capable(ns, cap); 1083 } 1084 EXPORT_SYMBOL(sockopt_ns_capable); 1085 1086 bool sockopt_capable(int cap) 1087 { 1088 return has_current_bpf_ctx() || capable(cap); 1089 } 1090 EXPORT_SYMBOL(sockopt_capable); 1091 1092 /* 1093 * This is meant for all protocols to use and covers goings on 1094 * at the socket level. Everything here is generic. 1095 */ 1096 1097 int sk_setsockopt(struct sock *sk, int level, int optname, 1098 sockptr_t optval, unsigned int optlen) 1099 { 1100 struct so_timestamping timestamping; 1101 struct socket *sock = sk->sk_socket; 1102 struct sock_txtime sk_txtime; 1103 int val; 1104 int valbool; 1105 struct linger ling; 1106 int ret = 0; 1107 1108 /* 1109 * Options without arguments 1110 */ 1111 1112 if (optname == SO_BINDTODEVICE) 1113 return sock_setbindtodevice(sk, optval, optlen); 1114 1115 if (optlen < sizeof(int)) 1116 return -EINVAL; 1117 1118 if (copy_from_sockptr(&val, optval, sizeof(val))) 1119 return -EFAULT; 1120 1121 valbool = val ? 1 : 0; 1122 1123 sockopt_lock_sock(sk); 1124 1125 switch (optname) { 1126 case SO_DEBUG: 1127 if (val && !sockopt_capable(CAP_NET_ADMIN)) 1128 ret = -EACCES; 1129 else 1130 sock_valbool_flag(sk, SOCK_DBG, valbool); 1131 break; 1132 case SO_REUSEADDR: 1133 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE); 1134 break; 1135 case SO_REUSEPORT: 1136 sk->sk_reuseport = valbool; 1137 break; 1138 case SO_TYPE: 1139 case SO_PROTOCOL: 1140 case SO_DOMAIN: 1141 case SO_ERROR: 1142 ret = -ENOPROTOOPT; 1143 break; 1144 case SO_DONTROUTE: 1145 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool); 1146 sk_dst_reset(sk); 1147 break; 1148 case SO_BROADCAST: 1149 sock_valbool_flag(sk, SOCK_BROADCAST, valbool); 1150 break; 1151 case SO_SNDBUF: 1152 /* Don't error on this BSD doesn't and if you think 1153 * about it this is right. Otherwise apps have to 1154 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1155 * are treated in BSD as hints 1156 */ 1157 val = min_t(u32, val, READ_ONCE(sysctl_wmem_max)); 1158 set_sndbuf: 1159 /* Ensure val * 2 fits into an int, to prevent max_t() 1160 * from treating it as a negative value. 1161 */ 1162 val = min_t(int, val, INT_MAX / 2); 1163 sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 1164 WRITE_ONCE(sk->sk_sndbuf, 1165 max_t(int, val * 2, SOCK_MIN_SNDBUF)); 1166 /* Wake up sending tasks if we upped the value. */ 1167 sk->sk_write_space(sk); 1168 break; 1169 1170 case SO_SNDBUFFORCE: 1171 if (!sockopt_capable(CAP_NET_ADMIN)) { 1172 ret = -EPERM; 1173 break; 1174 } 1175 1176 /* No negative values (to prevent underflow, as val will be 1177 * multiplied by 2). 1178 */ 1179 if (val < 0) 1180 val = 0; 1181 goto set_sndbuf; 1182 1183 case SO_RCVBUF: 1184 /* Don't error on this BSD doesn't and if you think 1185 * about it this is right. Otherwise apps have to 1186 * play 'guess the biggest size' games. RCVBUF/SNDBUF 1187 * are treated in BSD as hints 1188 */ 1189 __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max))); 1190 break; 1191 1192 case SO_RCVBUFFORCE: 1193 if (!sockopt_capable(CAP_NET_ADMIN)) { 1194 ret = -EPERM; 1195 break; 1196 } 1197 1198 /* No negative values (to prevent underflow, as val will be 1199 * multiplied by 2). 1200 */ 1201 __sock_set_rcvbuf(sk, max(val, 0)); 1202 break; 1203 1204 case SO_KEEPALIVE: 1205 if (sk->sk_prot->keepalive) 1206 sk->sk_prot->keepalive(sk, valbool); 1207 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 1208 break; 1209 1210 case SO_OOBINLINE: 1211 sock_valbool_flag(sk, SOCK_URGINLINE, valbool); 1212 break; 1213 1214 case SO_NO_CHECK: 1215 sk->sk_no_check_tx = valbool; 1216 break; 1217 1218 case SO_PRIORITY: 1219 if ((val >= 0 && val <= 6) || 1220 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || 1221 sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 1222 WRITE_ONCE(sk->sk_priority, val); 1223 else 1224 ret = -EPERM; 1225 break; 1226 1227 case SO_LINGER: 1228 if (optlen < sizeof(ling)) { 1229 ret = -EINVAL; /* 1003.1g */ 1230 break; 1231 } 1232 if (copy_from_sockptr(&ling, optval, sizeof(ling))) { 1233 ret = -EFAULT; 1234 break; 1235 } 1236 if (!ling.l_onoff) { 1237 sock_reset_flag(sk, SOCK_LINGER); 1238 } else { 1239 unsigned long t_sec = ling.l_linger; 1240 1241 if (t_sec >= MAX_SCHEDULE_TIMEOUT / HZ) 1242 WRITE_ONCE(sk->sk_lingertime, MAX_SCHEDULE_TIMEOUT); 1243 else 1244 WRITE_ONCE(sk->sk_lingertime, t_sec * HZ); 1245 sock_set_flag(sk, SOCK_LINGER); 1246 } 1247 break; 1248 1249 case SO_BSDCOMPAT: 1250 break; 1251 1252 case SO_PASSCRED: 1253 assign_bit(SOCK_PASSCRED, &sock->flags, valbool); 1254 break; 1255 1256 case SO_PASSPIDFD: 1257 assign_bit(SOCK_PASSPIDFD, &sock->flags, valbool); 1258 break; 1259 1260 case SO_TIMESTAMP_OLD: 1261 case SO_TIMESTAMP_NEW: 1262 case SO_TIMESTAMPNS_OLD: 1263 case SO_TIMESTAMPNS_NEW: 1264 sock_set_timestamp(sk, optname, valbool); 1265 break; 1266 1267 case SO_TIMESTAMPING_NEW: 1268 case SO_TIMESTAMPING_OLD: 1269 if (optlen == sizeof(timestamping)) { 1270 if (copy_from_sockptr(×tamping, optval, 1271 sizeof(timestamping))) { 1272 ret = -EFAULT; 1273 break; 1274 } 1275 } else { 1276 memset(×tamping, 0, sizeof(timestamping)); 1277 timestamping.flags = val; 1278 } 1279 ret = sock_set_timestamping(sk, optname, timestamping); 1280 break; 1281 1282 case SO_RCVLOWAT: 1283 { 1284 int (*set_rcvlowat)(struct sock *sk, int val) = NULL; 1285 1286 if (val < 0) 1287 val = INT_MAX; 1288 if (sock) 1289 set_rcvlowat = READ_ONCE(sock->ops)->set_rcvlowat; 1290 if (set_rcvlowat) 1291 ret = set_rcvlowat(sk, val); 1292 else 1293 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); 1294 break; 1295 } 1296 case SO_RCVTIMEO_OLD: 1297 case SO_RCVTIMEO_NEW: 1298 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, 1299 optlen, optname == SO_RCVTIMEO_OLD); 1300 break; 1301 1302 case SO_SNDTIMEO_OLD: 1303 case SO_SNDTIMEO_NEW: 1304 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, 1305 optlen, optname == SO_SNDTIMEO_OLD); 1306 break; 1307 1308 case SO_ATTACH_FILTER: { 1309 struct sock_fprog fprog; 1310 1311 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1312 if (!ret) 1313 ret = sk_attach_filter(&fprog, sk); 1314 break; 1315 } 1316 case SO_ATTACH_BPF: 1317 ret = -EINVAL; 1318 if (optlen == sizeof(u32)) { 1319 u32 ufd; 1320 1321 ret = -EFAULT; 1322 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1323 break; 1324 1325 ret = sk_attach_bpf(ufd, sk); 1326 } 1327 break; 1328 1329 case SO_ATTACH_REUSEPORT_CBPF: { 1330 struct sock_fprog fprog; 1331 1332 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); 1333 if (!ret) 1334 ret = sk_reuseport_attach_filter(&fprog, sk); 1335 break; 1336 } 1337 case SO_ATTACH_REUSEPORT_EBPF: 1338 ret = -EINVAL; 1339 if (optlen == sizeof(u32)) { 1340 u32 ufd; 1341 1342 ret = -EFAULT; 1343 if (copy_from_sockptr(&ufd, optval, sizeof(ufd))) 1344 break; 1345 1346 ret = sk_reuseport_attach_bpf(ufd, sk); 1347 } 1348 break; 1349 1350 case SO_DETACH_REUSEPORT_BPF: 1351 ret = reuseport_detach_prog(sk); 1352 break; 1353 1354 case SO_DETACH_FILTER: 1355 ret = sk_detach_filter(sk); 1356 break; 1357 1358 case SO_LOCK_FILTER: 1359 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool) 1360 ret = -EPERM; 1361 else 1362 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool); 1363 break; 1364 1365 case SO_PASSSEC: 1366 assign_bit(SOCK_PASSSEC, &sock->flags, valbool); 1367 break; 1368 case SO_MARK: 1369 if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 1370 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1371 ret = -EPERM; 1372 break; 1373 } 1374 1375 __sock_set_mark(sk, val); 1376 break; 1377 case SO_RCVMARK: 1378 sock_valbool_flag(sk, SOCK_RCVMARK, valbool); 1379 break; 1380 1381 case SO_RXQ_OVFL: 1382 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool); 1383 break; 1384 1385 case SO_WIFI_STATUS: 1386 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool); 1387 break; 1388 1389 case SO_PEEK_OFF: 1390 { 1391 int (*set_peek_off)(struct sock *sk, int val); 1392 1393 set_peek_off = READ_ONCE(sock->ops)->set_peek_off; 1394 if (set_peek_off) 1395 ret = set_peek_off(sk, val); 1396 else 1397 ret = -EOPNOTSUPP; 1398 break; 1399 } 1400 1401 case SO_NOFCS: 1402 sock_valbool_flag(sk, SOCK_NOFCS, valbool); 1403 break; 1404 1405 case SO_SELECT_ERR_QUEUE: 1406 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool); 1407 break; 1408 1409 #ifdef CONFIG_NET_RX_BUSY_POLL 1410 case SO_BUSY_POLL: 1411 if (val < 0) 1412 ret = -EINVAL; 1413 else 1414 WRITE_ONCE(sk->sk_ll_usec, val); 1415 break; 1416 case SO_PREFER_BUSY_POLL: 1417 if (valbool && !sockopt_capable(CAP_NET_ADMIN)) 1418 ret = -EPERM; 1419 else 1420 WRITE_ONCE(sk->sk_prefer_busy_poll, valbool); 1421 break; 1422 case SO_BUSY_POLL_BUDGET: 1423 if (val > READ_ONCE(sk->sk_busy_poll_budget) && !sockopt_capable(CAP_NET_ADMIN)) { 1424 ret = -EPERM; 1425 } else { 1426 if (val < 0 || val > U16_MAX) 1427 ret = -EINVAL; 1428 else 1429 WRITE_ONCE(sk->sk_busy_poll_budget, val); 1430 } 1431 break; 1432 #endif 1433 1434 case SO_MAX_PACING_RATE: 1435 { 1436 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val; 1437 1438 if (sizeof(ulval) != sizeof(val) && 1439 optlen >= sizeof(ulval) && 1440 copy_from_sockptr(&ulval, optval, sizeof(ulval))) { 1441 ret = -EFAULT; 1442 break; 1443 } 1444 if (ulval != ~0UL) 1445 cmpxchg(&sk->sk_pacing_status, 1446 SK_PACING_NONE, 1447 SK_PACING_NEEDED); 1448 /* Pairs with READ_ONCE() from sk_getsockopt() */ 1449 WRITE_ONCE(sk->sk_max_pacing_rate, ulval); 1450 sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval); 1451 break; 1452 } 1453 case SO_INCOMING_CPU: 1454 reuseport_update_incoming_cpu(sk, val); 1455 break; 1456 1457 case SO_CNX_ADVICE: 1458 if (val == 1) 1459 dst_negative_advice(sk); 1460 break; 1461 1462 case SO_ZEROCOPY: 1463 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { 1464 if (!(sk_is_tcp(sk) || 1465 (sk->sk_type == SOCK_DGRAM && 1466 sk->sk_protocol == IPPROTO_UDP))) 1467 ret = -EOPNOTSUPP; 1468 } else if (sk->sk_family != PF_RDS) { 1469 ret = -EOPNOTSUPP; 1470 } 1471 if (!ret) { 1472 if (val < 0 || val > 1) 1473 ret = -EINVAL; 1474 else 1475 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); 1476 } 1477 break; 1478 1479 case SO_TXTIME: 1480 if (optlen != sizeof(struct sock_txtime)) { 1481 ret = -EINVAL; 1482 break; 1483 } else if (copy_from_sockptr(&sk_txtime, optval, 1484 sizeof(struct sock_txtime))) { 1485 ret = -EFAULT; 1486 break; 1487 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) { 1488 ret = -EINVAL; 1489 break; 1490 } 1491 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet 1492 * scheduler has enough safe guards. 1493 */ 1494 if (sk_txtime.clockid != CLOCK_MONOTONIC && 1495 !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { 1496 ret = -EPERM; 1497 break; 1498 } 1499 sock_valbool_flag(sk, SOCK_TXTIME, true); 1500 sk->sk_clockid = sk_txtime.clockid; 1501 sk->sk_txtime_deadline_mode = 1502 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE); 1503 sk->sk_txtime_report_errors = 1504 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS); 1505 break; 1506 1507 case SO_BINDTOIFINDEX: 1508 ret = sock_bindtoindex_locked(sk, val); 1509 break; 1510 1511 case SO_BUF_LOCK: 1512 if (val & ~SOCK_BUF_LOCK_MASK) { 1513 ret = -EINVAL; 1514 break; 1515 } 1516 sk->sk_userlocks = val | (sk->sk_userlocks & 1517 ~SOCK_BUF_LOCK_MASK); 1518 break; 1519 1520 case SO_RESERVE_MEM: 1521 { 1522 int delta; 1523 1524 if (val < 0) { 1525 ret = -EINVAL; 1526 break; 1527 } 1528 1529 delta = val - sk->sk_reserved_mem; 1530 if (delta < 0) 1531 sock_release_reserved_memory(sk, -delta); 1532 else 1533 ret = sock_reserve_memory(sk, delta); 1534 break; 1535 } 1536 1537 case SO_TXREHASH: 1538 if (val < -1 || val > 1) { 1539 ret = -EINVAL; 1540 break; 1541 } 1542 if ((u8)val == SOCK_TXREHASH_DEFAULT) 1543 val = READ_ONCE(sock_net(sk)->core.sysctl_txrehash); 1544 /* Paired with READ_ONCE() in tcp_rtx_synack() 1545 * and sk_getsockopt(). 1546 */ 1547 WRITE_ONCE(sk->sk_txrehash, (u8)val); 1548 break; 1549 1550 default: 1551 ret = -ENOPROTOOPT; 1552 break; 1553 } 1554 sockopt_release_sock(sk); 1555 return ret; 1556 } 1557 1558 int sock_setsockopt(struct socket *sock, int level, int optname, 1559 sockptr_t optval, unsigned int optlen) 1560 { 1561 return sk_setsockopt(sock->sk, level, optname, 1562 optval, optlen); 1563 } 1564 EXPORT_SYMBOL(sock_setsockopt); 1565 1566 static const struct cred *sk_get_peer_cred(struct sock *sk) 1567 { 1568 const struct cred *cred; 1569 1570 spin_lock(&sk->sk_peer_lock); 1571 cred = get_cred(sk->sk_peer_cred); 1572 spin_unlock(&sk->sk_peer_lock); 1573 1574 return cred; 1575 } 1576 1577 static void cred_to_ucred(struct pid *pid, const struct cred *cred, 1578 struct ucred *ucred) 1579 { 1580 ucred->pid = pid_vnr(pid); 1581 ucred->uid = ucred->gid = -1; 1582 if (cred) { 1583 struct user_namespace *current_ns = current_user_ns(); 1584 1585 ucred->uid = from_kuid_munged(current_ns, cred->euid); 1586 ucred->gid = from_kgid_munged(current_ns, cred->egid); 1587 } 1588 } 1589 1590 static int groups_to_user(sockptr_t dst, const struct group_info *src) 1591 { 1592 struct user_namespace *user_ns = current_user_ns(); 1593 int i; 1594 1595 for (i = 0; i < src->ngroups; i++) { 1596 gid_t gid = from_kgid_munged(user_ns, src->gid[i]); 1597 1598 if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid))) 1599 return -EFAULT; 1600 } 1601 1602 return 0; 1603 } 1604 1605 int sk_getsockopt(struct sock *sk, int level, int optname, 1606 sockptr_t optval, sockptr_t optlen) 1607 { 1608 struct socket *sock = sk->sk_socket; 1609 1610 union { 1611 int val; 1612 u64 val64; 1613 unsigned long ulval; 1614 struct linger ling; 1615 struct old_timeval32 tm32; 1616 struct __kernel_old_timeval tm; 1617 struct __kernel_sock_timeval stm; 1618 struct sock_txtime txtime; 1619 struct so_timestamping timestamping; 1620 } v; 1621 1622 int lv = sizeof(int); 1623 int len; 1624 1625 if (copy_from_sockptr(&len, optlen, sizeof(int))) 1626 return -EFAULT; 1627 if (len < 0) 1628 return -EINVAL; 1629 1630 memset(&v, 0, sizeof(v)); 1631 1632 switch (optname) { 1633 case SO_DEBUG: 1634 v.val = sock_flag(sk, SOCK_DBG); 1635 break; 1636 1637 case SO_DONTROUTE: 1638 v.val = sock_flag(sk, SOCK_LOCALROUTE); 1639 break; 1640 1641 case SO_BROADCAST: 1642 v.val = sock_flag(sk, SOCK_BROADCAST); 1643 break; 1644 1645 case SO_SNDBUF: 1646 v.val = READ_ONCE(sk->sk_sndbuf); 1647 break; 1648 1649 case SO_RCVBUF: 1650 v.val = READ_ONCE(sk->sk_rcvbuf); 1651 break; 1652 1653 case SO_REUSEADDR: 1654 v.val = sk->sk_reuse; 1655 break; 1656 1657 case SO_REUSEPORT: 1658 v.val = sk->sk_reuseport; 1659 break; 1660 1661 case SO_KEEPALIVE: 1662 v.val = sock_flag(sk, SOCK_KEEPOPEN); 1663 break; 1664 1665 case SO_TYPE: 1666 v.val = sk->sk_type; 1667 break; 1668 1669 case SO_PROTOCOL: 1670 v.val = sk->sk_protocol; 1671 break; 1672 1673 case SO_DOMAIN: 1674 v.val = sk->sk_family; 1675 break; 1676 1677 case SO_ERROR: 1678 v.val = -sock_error(sk); 1679 if (v.val == 0) 1680 v.val = xchg(&sk->sk_err_soft, 0); 1681 break; 1682 1683 case SO_OOBINLINE: 1684 v.val = sock_flag(sk, SOCK_URGINLINE); 1685 break; 1686 1687 case SO_NO_CHECK: 1688 v.val = sk->sk_no_check_tx; 1689 break; 1690 1691 case SO_PRIORITY: 1692 v.val = READ_ONCE(sk->sk_priority); 1693 break; 1694 1695 case SO_LINGER: 1696 lv = sizeof(v.ling); 1697 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER); 1698 v.ling.l_linger = READ_ONCE(sk->sk_lingertime) / HZ; 1699 break; 1700 1701 case SO_BSDCOMPAT: 1702 break; 1703 1704 case SO_TIMESTAMP_OLD: 1705 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && 1706 !sock_flag(sk, SOCK_TSTAMP_NEW) && 1707 !sock_flag(sk, SOCK_RCVTSTAMPNS); 1708 break; 1709 1710 case SO_TIMESTAMPNS_OLD: 1711 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW); 1712 break; 1713 1714 case SO_TIMESTAMP_NEW: 1715 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW); 1716 break; 1717 1718 case SO_TIMESTAMPNS_NEW: 1719 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW); 1720 break; 1721 1722 case SO_TIMESTAMPING_OLD: 1723 case SO_TIMESTAMPING_NEW: 1724 lv = sizeof(v.timestamping); 1725 /* For the later-added case SO_TIMESTAMPING_NEW: Be strict about only 1726 * returning the flags when they were set through the same option. 1727 * Don't change the beviour for the old case SO_TIMESTAMPING_OLD. 1728 */ 1729 if (optname == SO_TIMESTAMPING_OLD || sock_flag(sk, SOCK_TSTAMP_NEW)) { 1730 v.timestamping.flags = READ_ONCE(sk->sk_tsflags); 1731 v.timestamping.bind_phc = READ_ONCE(sk->sk_bind_phc); 1732 } 1733 break; 1734 1735 case SO_RCVTIMEO_OLD: 1736 case SO_RCVTIMEO_NEW: 1737 lv = sock_get_timeout(READ_ONCE(sk->sk_rcvtimeo), &v, 1738 SO_RCVTIMEO_OLD == optname); 1739 break; 1740 1741 case SO_SNDTIMEO_OLD: 1742 case SO_SNDTIMEO_NEW: 1743 lv = sock_get_timeout(READ_ONCE(sk->sk_sndtimeo), &v, 1744 SO_SNDTIMEO_OLD == optname); 1745 break; 1746 1747 case SO_RCVLOWAT: 1748 v.val = READ_ONCE(sk->sk_rcvlowat); 1749 break; 1750 1751 case SO_SNDLOWAT: 1752 v.val = 1; 1753 break; 1754 1755 case SO_PASSCRED: 1756 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags); 1757 break; 1758 1759 case SO_PASSPIDFD: 1760 v.val = !!test_bit(SOCK_PASSPIDFD, &sock->flags); 1761 break; 1762 1763 case SO_PEERCRED: 1764 { 1765 struct ucred peercred; 1766 if (len > sizeof(peercred)) 1767 len = sizeof(peercred); 1768 1769 spin_lock(&sk->sk_peer_lock); 1770 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); 1771 spin_unlock(&sk->sk_peer_lock); 1772 1773 if (copy_to_sockptr(optval, &peercred, len)) 1774 return -EFAULT; 1775 goto lenout; 1776 } 1777 1778 case SO_PEERPIDFD: 1779 { 1780 struct pid *peer_pid; 1781 struct file *pidfd_file = NULL; 1782 int pidfd; 1783 1784 if (len > sizeof(pidfd)) 1785 len = sizeof(pidfd); 1786 1787 spin_lock(&sk->sk_peer_lock); 1788 peer_pid = get_pid(sk->sk_peer_pid); 1789 spin_unlock(&sk->sk_peer_lock); 1790 1791 if (!peer_pid) 1792 return -ENODATA; 1793 1794 pidfd = pidfd_prepare(peer_pid, 0, &pidfd_file); 1795 put_pid(peer_pid); 1796 if (pidfd < 0) 1797 return pidfd; 1798 1799 if (copy_to_sockptr(optval, &pidfd, len) || 1800 copy_to_sockptr(optlen, &len, sizeof(int))) { 1801 put_unused_fd(pidfd); 1802 fput(pidfd_file); 1803 1804 return -EFAULT; 1805 } 1806 1807 fd_install(pidfd, pidfd_file); 1808 return 0; 1809 } 1810 1811 case SO_PEERGROUPS: 1812 { 1813 const struct cred *cred; 1814 int ret, n; 1815 1816 cred = sk_get_peer_cred(sk); 1817 if (!cred) 1818 return -ENODATA; 1819 1820 n = cred->group_info->ngroups; 1821 if (len < n * sizeof(gid_t)) { 1822 len = n * sizeof(gid_t); 1823 put_cred(cred); 1824 return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE; 1825 } 1826 len = n * sizeof(gid_t); 1827 1828 ret = groups_to_user(optval, cred->group_info); 1829 put_cred(cred); 1830 if (ret) 1831 return ret; 1832 goto lenout; 1833 } 1834 1835 case SO_PEERNAME: 1836 { 1837 struct sockaddr_storage address; 1838 1839 lv = READ_ONCE(sock->ops)->getname(sock, (struct sockaddr *)&address, 2); 1840 if (lv < 0) 1841 return -ENOTCONN; 1842 if (lv < len) 1843 return -EINVAL; 1844 if (copy_to_sockptr(optval, &address, len)) 1845 return -EFAULT; 1846 goto lenout; 1847 } 1848 1849 /* Dubious BSD thing... Probably nobody even uses it, but 1850 * the UNIX standard wants it for whatever reason... -DaveM 1851 */ 1852 case SO_ACCEPTCONN: 1853 v.val = sk->sk_state == TCP_LISTEN; 1854 break; 1855 1856 case SO_PASSSEC: 1857 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags); 1858 break; 1859 1860 case SO_PEERSEC: 1861 return security_socket_getpeersec_stream(sock, 1862 optval, optlen, len); 1863 1864 case SO_MARK: 1865 v.val = READ_ONCE(sk->sk_mark); 1866 break; 1867 1868 case SO_RCVMARK: 1869 v.val = sock_flag(sk, SOCK_RCVMARK); 1870 break; 1871 1872 case SO_RXQ_OVFL: 1873 v.val = sock_flag(sk, SOCK_RXQ_OVFL); 1874 break; 1875 1876 case SO_WIFI_STATUS: 1877 v.val = sock_flag(sk, SOCK_WIFI_STATUS); 1878 break; 1879 1880 case SO_PEEK_OFF: 1881 if (!READ_ONCE(sock->ops)->set_peek_off) 1882 return -EOPNOTSUPP; 1883 1884 v.val = READ_ONCE(sk->sk_peek_off); 1885 break; 1886 case SO_NOFCS: 1887 v.val = sock_flag(sk, SOCK_NOFCS); 1888 break; 1889 1890 case SO_BINDTODEVICE: 1891 return sock_getbindtodevice(sk, optval, optlen, len); 1892 1893 case SO_GET_FILTER: 1894 len = sk_get_filter(sk, optval, len); 1895 if (len < 0) 1896 return len; 1897 1898 goto lenout; 1899 1900 case SO_LOCK_FILTER: 1901 v.val = sock_flag(sk, SOCK_FILTER_LOCKED); 1902 break; 1903 1904 case SO_BPF_EXTENSIONS: 1905 v.val = bpf_tell_extensions(); 1906 break; 1907 1908 case SO_SELECT_ERR_QUEUE: 1909 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE); 1910 break; 1911 1912 #ifdef CONFIG_NET_RX_BUSY_POLL 1913 case SO_BUSY_POLL: 1914 v.val = READ_ONCE(sk->sk_ll_usec); 1915 break; 1916 case SO_PREFER_BUSY_POLL: 1917 v.val = READ_ONCE(sk->sk_prefer_busy_poll); 1918 break; 1919 #endif 1920 1921 case SO_MAX_PACING_RATE: 1922 /* The READ_ONCE() pair with the WRITE_ONCE() in sk_setsockopt() */ 1923 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) { 1924 lv = sizeof(v.ulval); 1925 v.ulval = READ_ONCE(sk->sk_max_pacing_rate); 1926 } else { 1927 /* 32bit version */ 1928 v.val = min_t(unsigned long, ~0U, 1929 READ_ONCE(sk->sk_max_pacing_rate)); 1930 } 1931 break; 1932 1933 case SO_INCOMING_CPU: 1934 v.val = READ_ONCE(sk->sk_incoming_cpu); 1935 break; 1936 1937 case SO_MEMINFO: 1938 { 1939 u32 meminfo[SK_MEMINFO_VARS]; 1940 1941 sk_get_meminfo(sk, meminfo); 1942 1943 len = min_t(unsigned int, len, sizeof(meminfo)); 1944 if (copy_to_sockptr(optval, &meminfo, len)) 1945 return -EFAULT; 1946 1947 goto lenout; 1948 } 1949 1950 #ifdef CONFIG_NET_RX_BUSY_POLL 1951 case SO_INCOMING_NAPI_ID: 1952 v.val = READ_ONCE(sk->sk_napi_id); 1953 1954 /* aggregate non-NAPI IDs down to 0 */ 1955 if (v.val < MIN_NAPI_ID) 1956 v.val = 0; 1957 1958 break; 1959 #endif 1960 1961 case SO_COOKIE: 1962 lv = sizeof(u64); 1963 if (len < lv) 1964 return -EINVAL; 1965 v.val64 = sock_gen_cookie(sk); 1966 break; 1967 1968 case SO_ZEROCOPY: 1969 v.val = sock_flag(sk, SOCK_ZEROCOPY); 1970 break; 1971 1972 case SO_TXTIME: 1973 lv = sizeof(v.txtime); 1974 v.txtime.clockid = sk->sk_clockid; 1975 v.txtime.flags |= sk->sk_txtime_deadline_mode ? 1976 SOF_TXTIME_DEADLINE_MODE : 0; 1977 v.txtime.flags |= sk->sk_txtime_report_errors ? 1978 SOF_TXTIME_REPORT_ERRORS : 0; 1979 break; 1980 1981 case SO_BINDTOIFINDEX: 1982 v.val = READ_ONCE(sk->sk_bound_dev_if); 1983 break; 1984 1985 case SO_NETNS_COOKIE: 1986 lv = sizeof(u64); 1987 if (len != lv) 1988 return -EINVAL; 1989 v.val64 = sock_net(sk)->net_cookie; 1990 break; 1991 1992 case SO_BUF_LOCK: 1993 v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK; 1994 break; 1995 1996 case SO_RESERVE_MEM: 1997 v.val = READ_ONCE(sk->sk_reserved_mem); 1998 break; 1999 2000 case SO_TXREHASH: 2001 /* Paired with WRITE_ONCE() in sk_setsockopt() */ 2002 v.val = READ_ONCE(sk->sk_txrehash); 2003 break; 2004 2005 default: 2006 /* We implement the SO_SNDLOWAT etc to not be settable 2007 * (1003.1g 7). 2008 */ 2009 return -ENOPROTOOPT; 2010 } 2011 2012 if (len > lv) 2013 len = lv; 2014 if (copy_to_sockptr(optval, &v, len)) 2015 return -EFAULT; 2016 lenout: 2017 if (copy_to_sockptr(optlen, &len, sizeof(int))) 2018 return -EFAULT; 2019 return 0; 2020 } 2021 2022 int sock_getsockopt(struct socket *sock, int level, int optname, 2023 char __user *optval, int __user *optlen) 2024 { 2025 return sk_getsockopt(sock->sk, level, optname, 2026 USER_SOCKPTR(optval), 2027 USER_SOCKPTR(optlen)); 2028 } 2029 2030 /* 2031 * Initialize an sk_lock. 2032 * 2033 * (We also register the sk_lock with the lock validator.) 2034 */ 2035 static inline void sock_lock_init(struct sock *sk) 2036 { 2037 if (sk->sk_kern_sock) 2038 sock_lock_init_class_and_name( 2039 sk, 2040 af_family_kern_slock_key_strings[sk->sk_family], 2041 af_family_kern_slock_keys + sk->sk_family, 2042 af_family_kern_key_strings[sk->sk_family], 2043 af_family_kern_keys + sk->sk_family); 2044 else 2045 sock_lock_init_class_and_name( 2046 sk, 2047 af_family_slock_key_strings[sk->sk_family], 2048 af_family_slock_keys + sk->sk_family, 2049 af_family_key_strings[sk->sk_family], 2050 af_family_keys + sk->sk_family); 2051 } 2052 2053 /* 2054 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet, 2055 * even temporarly, because of RCU lookups. sk_node should also be left as is. 2056 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end 2057 */ 2058 static void sock_copy(struct sock *nsk, const struct sock *osk) 2059 { 2060 const struct proto *prot = READ_ONCE(osk->sk_prot); 2061 #ifdef CONFIG_SECURITY_NETWORK 2062 void *sptr = nsk->sk_security; 2063 #endif 2064 2065 /* If we move sk_tx_queue_mapping out of the private section, 2066 * we must check if sk_tx_queue_clear() is called after 2067 * sock_copy() in sk_clone_lock(). 2068 */ 2069 BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) < 2070 offsetof(struct sock, sk_dontcopy_begin) || 2071 offsetof(struct sock, sk_tx_queue_mapping) >= 2072 offsetof(struct sock, sk_dontcopy_end)); 2073 2074 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin)); 2075 2076 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end, 2077 prot->obj_size - offsetof(struct sock, sk_dontcopy_end)); 2078 2079 #ifdef CONFIG_SECURITY_NETWORK 2080 nsk->sk_security = sptr; 2081 security_sk_clone(osk, nsk); 2082 #endif 2083 } 2084 2085 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, 2086 int family) 2087 { 2088 struct sock *sk; 2089 struct kmem_cache *slab; 2090 2091 slab = prot->slab; 2092 if (slab != NULL) { 2093 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO); 2094 if (!sk) 2095 return sk; 2096 if (want_init_on_alloc(priority)) 2097 sk_prot_clear_nulls(sk, prot->obj_size); 2098 } else 2099 sk = kmalloc(prot->obj_size, priority); 2100 2101 if (sk != NULL) { 2102 if (security_sk_alloc(sk, family, priority)) 2103 goto out_free; 2104 2105 if (!try_module_get(prot->owner)) 2106 goto out_free_sec; 2107 } 2108 2109 return sk; 2110 2111 out_free_sec: 2112 security_sk_free(sk); 2113 out_free: 2114 if (slab != NULL) 2115 kmem_cache_free(slab, sk); 2116 else 2117 kfree(sk); 2118 return NULL; 2119 } 2120 2121 static void sk_prot_free(struct proto *prot, struct sock *sk) 2122 { 2123 struct kmem_cache *slab; 2124 struct module *owner; 2125 2126 owner = prot->owner; 2127 slab = prot->slab; 2128 2129 cgroup_sk_free(&sk->sk_cgrp_data); 2130 mem_cgroup_sk_free(sk); 2131 security_sk_free(sk); 2132 if (slab != NULL) 2133 kmem_cache_free(slab, sk); 2134 else 2135 kfree(sk); 2136 module_put(owner); 2137 } 2138 2139 /** 2140 * sk_alloc - All socket objects are allocated here 2141 * @net: the applicable net namespace 2142 * @family: protocol family 2143 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2144 * @prot: struct proto associated with this new sock instance 2145 * @kern: is this to be a kernel socket? 2146 */ 2147 struct sock *sk_alloc(struct net *net, int family, gfp_t priority, 2148 struct proto *prot, int kern) 2149 { 2150 struct sock *sk; 2151 2152 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family); 2153 if (sk) { 2154 sk->sk_family = family; 2155 /* 2156 * See comment in struct sock definition to understand 2157 * why we need sk_prot_creator -acme 2158 */ 2159 sk->sk_prot = sk->sk_prot_creator = prot; 2160 sk->sk_kern_sock = kern; 2161 sock_lock_init(sk); 2162 sk->sk_net_refcnt = kern ? 0 : 1; 2163 if (likely(sk->sk_net_refcnt)) { 2164 get_net_track(net, &sk->ns_tracker, priority); 2165 sock_inuse_add(net, 1); 2166 } else { 2167 __netns_tracker_alloc(net, &sk->ns_tracker, 2168 false, priority); 2169 } 2170 2171 sock_net_set(sk, net); 2172 refcount_set(&sk->sk_wmem_alloc, 1); 2173 2174 mem_cgroup_sk_alloc(sk); 2175 cgroup_sk_alloc(&sk->sk_cgrp_data); 2176 sock_update_classid(&sk->sk_cgrp_data); 2177 sock_update_netprioidx(&sk->sk_cgrp_data); 2178 sk_tx_queue_clear(sk); 2179 } 2180 2181 return sk; 2182 } 2183 EXPORT_SYMBOL(sk_alloc); 2184 2185 /* Sockets having SOCK_RCU_FREE will call this function after one RCU 2186 * grace period. This is the case for UDP sockets and TCP listeners. 2187 */ 2188 static void __sk_destruct(struct rcu_head *head) 2189 { 2190 struct sock *sk = container_of(head, struct sock, sk_rcu); 2191 struct sk_filter *filter; 2192 2193 if (sk->sk_destruct) 2194 sk->sk_destruct(sk); 2195 2196 filter = rcu_dereference_check(sk->sk_filter, 2197 refcount_read(&sk->sk_wmem_alloc) == 0); 2198 if (filter) { 2199 sk_filter_uncharge(sk, filter); 2200 RCU_INIT_POINTER(sk->sk_filter, NULL); 2201 } 2202 2203 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); 2204 2205 #ifdef CONFIG_BPF_SYSCALL 2206 bpf_sk_storage_free(sk); 2207 #endif 2208 2209 if (atomic_read(&sk->sk_omem_alloc)) 2210 pr_debug("%s: optmem leakage (%d bytes) detected\n", 2211 __func__, atomic_read(&sk->sk_omem_alloc)); 2212 2213 if (sk->sk_frag.page) { 2214 put_page(sk->sk_frag.page); 2215 sk->sk_frag.page = NULL; 2216 } 2217 2218 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */ 2219 put_cred(sk->sk_peer_cred); 2220 put_pid(sk->sk_peer_pid); 2221 2222 if (likely(sk->sk_net_refcnt)) 2223 put_net_track(sock_net(sk), &sk->ns_tracker); 2224 else 2225 __netns_tracker_free(sock_net(sk), &sk->ns_tracker, false); 2226 2227 sk_prot_free(sk->sk_prot_creator, sk); 2228 } 2229 2230 void sk_destruct(struct sock *sk) 2231 { 2232 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE); 2233 2234 if (rcu_access_pointer(sk->sk_reuseport_cb)) { 2235 reuseport_detach_sock(sk); 2236 use_call_rcu = true; 2237 } 2238 2239 if (use_call_rcu) 2240 call_rcu(&sk->sk_rcu, __sk_destruct); 2241 else 2242 __sk_destruct(&sk->sk_rcu); 2243 } 2244 2245 static void __sk_free(struct sock *sk) 2246 { 2247 if (likely(sk->sk_net_refcnt)) 2248 sock_inuse_add(sock_net(sk), -1); 2249 2250 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk))) 2251 sock_diag_broadcast_destroy(sk); 2252 else 2253 sk_destruct(sk); 2254 } 2255 2256 void sk_free(struct sock *sk) 2257 { 2258 /* 2259 * We subtract one from sk_wmem_alloc and can know if 2260 * some packets are still in some tx queue. 2261 * If not null, sock_wfree() will call __sk_free(sk) later 2262 */ 2263 if (refcount_dec_and_test(&sk->sk_wmem_alloc)) 2264 __sk_free(sk); 2265 } 2266 EXPORT_SYMBOL(sk_free); 2267 2268 static void sk_init_common(struct sock *sk) 2269 { 2270 skb_queue_head_init(&sk->sk_receive_queue); 2271 skb_queue_head_init(&sk->sk_write_queue); 2272 skb_queue_head_init(&sk->sk_error_queue); 2273 2274 rwlock_init(&sk->sk_callback_lock); 2275 lockdep_set_class_and_name(&sk->sk_receive_queue.lock, 2276 af_rlock_keys + sk->sk_family, 2277 af_family_rlock_key_strings[sk->sk_family]); 2278 lockdep_set_class_and_name(&sk->sk_write_queue.lock, 2279 af_wlock_keys + sk->sk_family, 2280 af_family_wlock_key_strings[sk->sk_family]); 2281 lockdep_set_class_and_name(&sk->sk_error_queue.lock, 2282 af_elock_keys + sk->sk_family, 2283 af_family_elock_key_strings[sk->sk_family]); 2284 lockdep_set_class_and_name(&sk->sk_callback_lock, 2285 af_callback_keys + sk->sk_family, 2286 af_family_clock_key_strings[sk->sk_family]); 2287 } 2288 2289 /** 2290 * sk_clone_lock - clone a socket, and lock its clone 2291 * @sk: the socket to clone 2292 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 2293 * 2294 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 2295 */ 2296 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) 2297 { 2298 struct proto *prot = READ_ONCE(sk->sk_prot); 2299 struct sk_filter *filter; 2300 bool is_charged = true; 2301 struct sock *newsk; 2302 2303 newsk = sk_prot_alloc(prot, priority, sk->sk_family); 2304 if (!newsk) 2305 goto out; 2306 2307 sock_copy(newsk, sk); 2308 2309 newsk->sk_prot_creator = prot; 2310 2311 /* SANITY */ 2312 if (likely(newsk->sk_net_refcnt)) { 2313 get_net_track(sock_net(newsk), &newsk->ns_tracker, priority); 2314 sock_inuse_add(sock_net(newsk), 1); 2315 } else { 2316 /* Kernel sockets are not elevating the struct net refcount. 2317 * Instead, use a tracker to more easily detect if a layer 2318 * is not properly dismantling its kernel sockets at netns 2319 * destroy time. 2320 */ 2321 __netns_tracker_alloc(sock_net(newsk), &newsk->ns_tracker, 2322 false, priority); 2323 } 2324 sk_node_init(&newsk->sk_node); 2325 sock_lock_init(newsk); 2326 bh_lock_sock(newsk); 2327 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL; 2328 newsk->sk_backlog.len = 0; 2329 2330 atomic_set(&newsk->sk_rmem_alloc, 0); 2331 2332 /* sk_wmem_alloc set to one (see sk_free() and sock_wfree()) */ 2333 refcount_set(&newsk->sk_wmem_alloc, 1); 2334 2335 atomic_set(&newsk->sk_omem_alloc, 0); 2336 sk_init_common(newsk); 2337 2338 newsk->sk_dst_cache = NULL; 2339 newsk->sk_dst_pending_confirm = 0; 2340 newsk->sk_wmem_queued = 0; 2341 newsk->sk_forward_alloc = 0; 2342 newsk->sk_reserved_mem = 0; 2343 atomic_set(&newsk->sk_drops, 0); 2344 newsk->sk_send_head = NULL; 2345 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK; 2346 atomic_set(&newsk->sk_zckey, 0); 2347 2348 sock_reset_flag(newsk, SOCK_DONE); 2349 2350 /* sk->sk_memcg will be populated at accept() time */ 2351 newsk->sk_memcg = NULL; 2352 2353 cgroup_sk_clone(&newsk->sk_cgrp_data); 2354 2355 rcu_read_lock(); 2356 filter = rcu_dereference(sk->sk_filter); 2357 if (filter != NULL) 2358 /* though it's an empty new sock, the charging may fail 2359 * if sysctl_optmem_max was changed between creation of 2360 * original socket and cloning 2361 */ 2362 is_charged = sk_filter_charge(newsk, filter); 2363 RCU_INIT_POINTER(newsk->sk_filter, filter); 2364 rcu_read_unlock(); 2365 2366 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) { 2367 /* We need to make sure that we don't uncharge the new 2368 * socket if we couldn't charge it in the first place 2369 * as otherwise we uncharge the parent's filter. 2370 */ 2371 if (!is_charged) 2372 RCU_INIT_POINTER(newsk->sk_filter, NULL); 2373 sk_free_unlock_clone(newsk); 2374 newsk = NULL; 2375 goto out; 2376 } 2377 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); 2378 2379 if (bpf_sk_storage_clone(sk, newsk)) { 2380 sk_free_unlock_clone(newsk); 2381 newsk = NULL; 2382 goto out; 2383 } 2384 2385 /* Clear sk_user_data if parent had the pointer tagged 2386 * as not suitable for copying when cloning. 2387 */ 2388 if (sk_user_data_is_nocopy(newsk)) 2389 newsk->sk_user_data = NULL; 2390 2391 newsk->sk_err = 0; 2392 newsk->sk_err_soft = 0; 2393 newsk->sk_priority = 0; 2394 newsk->sk_incoming_cpu = raw_smp_processor_id(); 2395 2396 /* Before updating sk_refcnt, we must commit prior changes to memory 2397 * (Documentation/RCU/rculist_nulls.rst for details) 2398 */ 2399 smp_wmb(); 2400 refcount_set(&newsk->sk_refcnt, 2); 2401 2402 sk_set_socket(newsk, NULL); 2403 sk_tx_queue_clear(newsk); 2404 RCU_INIT_POINTER(newsk->sk_wq, NULL); 2405 2406 if (newsk->sk_prot->sockets_allocated) 2407 sk_sockets_allocated_inc(newsk); 2408 2409 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) 2410 net_enable_timestamp(); 2411 out: 2412 return newsk; 2413 } 2414 EXPORT_SYMBOL_GPL(sk_clone_lock); 2415 2416 void sk_free_unlock_clone(struct sock *sk) 2417 { 2418 /* It is still raw copy of parent, so invalidate 2419 * destructor and make plain sk_free() */ 2420 sk->sk_destruct = NULL; 2421 bh_unlock_sock(sk); 2422 sk_free(sk); 2423 } 2424 EXPORT_SYMBOL_GPL(sk_free_unlock_clone); 2425 2426 static u32 sk_dst_gso_max_size(struct sock *sk, struct dst_entry *dst) 2427 { 2428 bool is_ipv6 = false; 2429 u32 max_size; 2430 2431 #if IS_ENABLED(CONFIG_IPV6) 2432 is_ipv6 = (sk->sk_family == AF_INET6 && 2433 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)); 2434 #endif 2435 /* pairs with the WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 2436 max_size = is_ipv6 ? READ_ONCE(dst->dev->gso_max_size) : 2437 READ_ONCE(dst->dev->gso_ipv4_max_size); 2438 if (max_size > GSO_LEGACY_MAX_SIZE && !sk_is_tcp(sk)) 2439 max_size = GSO_LEGACY_MAX_SIZE; 2440 2441 return max_size - (MAX_TCP_HEADER + 1); 2442 } 2443 2444 void sk_setup_caps(struct sock *sk, struct dst_entry *dst) 2445 { 2446 u32 max_segs = 1; 2447 2448 sk->sk_route_caps = dst->dev->features; 2449 if (sk_is_tcp(sk)) 2450 sk->sk_route_caps |= NETIF_F_GSO; 2451 if (sk->sk_route_caps & NETIF_F_GSO) 2452 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; 2453 if (unlikely(sk->sk_gso_disabled)) 2454 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2455 if (sk_can_gso(sk)) { 2456 if (dst->header_len && !xfrm_dst_offload_ok(dst)) { 2457 sk->sk_route_caps &= ~NETIF_F_GSO_MASK; 2458 } else { 2459 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; 2460 sk->sk_gso_max_size = sk_dst_gso_max_size(sk, dst); 2461 /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ 2462 max_segs = max_t(u32, READ_ONCE(dst->dev->gso_max_segs), 1); 2463 } 2464 } 2465 sk->sk_gso_max_segs = max_segs; 2466 sk_dst_set(sk, dst); 2467 } 2468 EXPORT_SYMBOL_GPL(sk_setup_caps); 2469 2470 /* 2471 * Simple resource managers for sockets. 2472 */ 2473 2474 2475 /* 2476 * Write buffer destructor automatically called from kfree_skb. 2477 */ 2478 void sock_wfree(struct sk_buff *skb) 2479 { 2480 struct sock *sk = skb->sk; 2481 unsigned int len = skb->truesize; 2482 bool free; 2483 2484 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { 2485 if (sock_flag(sk, SOCK_RCU_FREE) && 2486 sk->sk_write_space == sock_def_write_space) { 2487 rcu_read_lock(); 2488 free = refcount_sub_and_test(len, &sk->sk_wmem_alloc); 2489 sock_def_write_space_wfree(sk); 2490 rcu_read_unlock(); 2491 if (unlikely(free)) 2492 __sk_free(sk); 2493 return; 2494 } 2495 2496 /* 2497 * Keep a reference on sk_wmem_alloc, this will be released 2498 * after sk_write_space() call 2499 */ 2500 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); 2501 sk->sk_write_space(sk); 2502 len = 1; 2503 } 2504 /* 2505 * if sk_wmem_alloc reaches 0, we must finish what sk_free() 2506 * could not do because of in-flight packets 2507 */ 2508 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc)) 2509 __sk_free(sk); 2510 } 2511 EXPORT_SYMBOL(sock_wfree); 2512 2513 /* This variant of sock_wfree() is used by TCP, 2514 * since it sets SOCK_USE_WRITE_QUEUE. 2515 */ 2516 void __sock_wfree(struct sk_buff *skb) 2517 { 2518 struct sock *sk = skb->sk; 2519 2520 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) 2521 __sk_free(sk); 2522 } 2523 2524 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) 2525 { 2526 skb_orphan(skb); 2527 skb->sk = sk; 2528 #ifdef CONFIG_INET 2529 if (unlikely(!sk_fullsock(sk))) { 2530 skb->destructor = sock_edemux; 2531 sock_hold(sk); 2532 return; 2533 } 2534 #endif 2535 skb->destructor = sock_wfree; 2536 skb_set_hash_from_sk(skb, sk); 2537 /* 2538 * We used to take a refcount on sk, but following operation 2539 * is enough to guarantee sk_free() wont free this sock until 2540 * all in-flight packets are completed 2541 */ 2542 refcount_add(skb->truesize, &sk->sk_wmem_alloc); 2543 } 2544 EXPORT_SYMBOL(skb_set_owner_w); 2545 2546 static bool can_skb_orphan_partial(const struct sk_buff *skb) 2547 { 2548 #ifdef CONFIG_TLS_DEVICE 2549 /* Drivers depend on in-order delivery for crypto offload, 2550 * partial orphan breaks out-of-order-OK logic. 2551 */ 2552 if (skb->decrypted) 2553 return false; 2554 #endif 2555 return (skb->destructor == sock_wfree || 2556 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree)); 2557 } 2558 2559 /* This helper is used by netem, as it can hold packets in its 2560 * delay queue. We want to allow the owner socket to send more 2561 * packets, as if they were already TX completed by a typical driver. 2562 * But we also want to keep skb->sk set because some packet schedulers 2563 * rely on it (sch_fq for example). 2564 */ 2565 void skb_orphan_partial(struct sk_buff *skb) 2566 { 2567 if (skb_is_tcp_pure_ack(skb)) 2568 return; 2569 2570 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk)) 2571 return; 2572 2573 skb_orphan(skb); 2574 } 2575 EXPORT_SYMBOL(skb_orphan_partial); 2576 2577 /* 2578 * Read buffer destructor automatically called from kfree_skb. 2579 */ 2580 void sock_rfree(struct sk_buff *skb) 2581 { 2582 struct sock *sk = skb->sk; 2583 unsigned int len = skb->truesize; 2584 2585 atomic_sub(len, &sk->sk_rmem_alloc); 2586 sk_mem_uncharge(sk, len); 2587 } 2588 EXPORT_SYMBOL(sock_rfree); 2589 2590 /* 2591 * Buffer destructor for skbs that are not used directly in read or write 2592 * path, e.g. for error handler skbs. Automatically called from kfree_skb. 2593 */ 2594 void sock_efree(struct sk_buff *skb) 2595 { 2596 sock_put(skb->sk); 2597 } 2598 EXPORT_SYMBOL(sock_efree); 2599 2600 /* Buffer destructor for prefetch/receive path where reference count may 2601 * not be held, e.g. for listen sockets. 2602 */ 2603 #ifdef CONFIG_INET 2604 void sock_pfree(struct sk_buff *skb) 2605 { 2606 if (sk_is_refcounted(skb->sk)) 2607 sock_gen_put(skb->sk); 2608 } 2609 EXPORT_SYMBOL(sock_pfree); 2610 #endif /* CONFIG_INET */ 2611 2612 kuid_t sock_i_uid(struct sock *sk) 2613 { 2614 kuid_t uid; 2615 2616 read_lock_bh(&sk->sk_callback_lock); 2617 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID; 2618 read_unlock_bh(&sk->sk_callback_lock); 2619 return uid; 2620 } 2621 EXPORT_SYMBOL(sock_i_uid); 2622 2623 unsigned long __sock_i_ino(struct sock *sk) 2624 { 2625 unsigned long ino; 2626 2627 read_lock(&sk->sk_callback_lock); 2628 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0; 2629 read_unlock(&sk->sk_callback_lock); 2630 return ino; 2631 } 2632 EXPORT_SYMBOL(__sock_i_ino); 2633 2634 unsigned long sock_i_ino(struct sock *sk) 2635 { 2636 unsigned long ino; 2637 2638 local_bh_disable(); 2639 ino = __sock_i_ino(sk); 2640 local_bh_enable(); 2641 return ino; 2642 } 2643 EXPORT_SYMBOL(sock_i_ino); 2644 2645 /* 2646 * Allocate a skb from the socket's send buffer. 2647 */ 2648 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, 2649 gfp_t priority) 2650 { 2651 if (force || 2652 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) { 2653 struct sk_buff *skb = alloc_skb(size, priority); 2654 2655 if (skb) { 2656 skb_set_owner_w(skb, sk); 2657 return skb; 2658 } 2659 } 2660 return NULL; 2661 } 2662 EXPORT_SYMBOL(sock_wmalloc); 2663 2664 static void sock_ofree(struct sk_buff *skb) 2665 { 2666 struct sock *sk = skb->sk; 2667 2668 atomic_sub(skb->truesize, &sk->sk_omem_alloc); 2669 } 2670 2671 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, 2672 gfp_t priority) 2673 { 2674 struct sk_buff *skb; 2675 2676 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ 2677 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > 2678 READ_ONCE(sysctl_optmem_max)) 2679 return NULL; 2680 2681 skb = alloc_skb(size, priority); 2682 if (!skb) 2683 return NULL; 2684 2685 atomic_add(skb->truesize, &sk->sk_omem_alloc); 2686 skb->sk = sk; 2687 skb->destructor = sock_ofree; 2688 return skb; 2689 } 2690 2691 /* 2692 * Allocate a memory block from the socket's option memory buffer. 2693 */ 2694 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) 2695 { 2696 int optmem_max = READ_ONCE(sysctl_optmem_max); 2697 2698 if ((unsigned int)size <= optmem_max && 2699 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 2700 void *mem; 2701 /* First do the add, to avoid the race if kmalloc 2702 * might sleep. 2703 */ 2704 atomic_add(size, &sk->sk_omem_alloc); 2705 mem = kmalloc(size, priority); 2706 if (mem) 2707 return mem; 2708 atomic_sub(size, &sk->sk_omem_alloc); 2709 } 2710 return NULL; 2711 } 2712 EXPORT_SYMBOL(sock_kmalloc); 2713 2714 /* Free an option memory block. Note, we actually want the inline 2715 * here as this allows gcc to detect the nullify and fold away the 2716 * condition entirely. 2717 */ 2718 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size, 2719 const bool nullify) 2720 { 2721 if (WARN_ON_ONCE(!mem)) 2722 return; 2723 if (nullify) 2724 kfree_sensitive(mem); 2725 else 2726 kfree(mem); 2727 atomic_sub(size, &sk->sk_omem_alloc); 2728 } 2729 2730 void sock_kfree_s(struct sock *sk, void *mem, int size) 2731 { 2732 __sock_kfree_s(sk, mem, size, false); 2733 } 2734 EXPORT_SYMBOL(sock_kfree_s); 2735 2736 void sock_kzfree_s(struct sock *sk, void *mem, int size) 2737 { 2738 __sock_kfree_s(sk, mem, size, true); 2739 } 2740 EXPORT_SYMBOL(sock_kzfree_s); 2741 2742 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock. 2743 I think, these locks should be removed for datagram sockets. 2744 */ 2745 static long sock_wait_for_wmem(struct sock *sk, long timeo) 2746 { 2747 DEFINE_WAIT(wait); 2748 2749 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2750 for (;;) { 2751 if (!timeo) 2752 break; 2753 if (signal_pending(current)) 2754 break; 2755 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2756 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 2757 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) 2758 break; 2759 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2760 break; 2761 if (READ_ONCE(sk->sk_err)) 2762 break; 2763 timeo = schedule_timeout(timeo); 2764 } 2765 finish_wait(sk_sleep(sk), &wait); 2766 return timeo; 2767 } 2768 2769 2770 /* 2771 * Generic send/receive buffer handlers 2772 */ 2773 2774 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, 2775 unsigned long data_len, int noblock, 2776 int *errcode, int max_page_order) 2777 { 2778 struct sk_buff *skb; 2779 long timeo; 2780 int err; 2781 2782 timeo = sock_sndtimeo(sk, noblock); 2783 for (;;) { 2784 err = sock_error(sk); 2785 if (err != 0) 2786 goto failure; 2787 2788 err = -EPIPE; 2789 if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) 2790 goto failure; 2791 2792 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf)) 2793 break; 2794 2795 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 2796 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 2797 err = -EAGAIN; 2798 if (!timeo) 2799 goto failure; 2800 if (signal_pending(current)) 2801 goto interrupted; 2802 timeo = sock_wait_for_wmem(sk, timeo); 2803 } 2804 skb = alloc_skb_with_frags(header_len, data_len, max_page_order, 2805 errcode, sk->sk_allocation); 2806 if (skb) 2807 skb_set_owner_w(skb, sk); 2808 return skb; 2809 2810 interrupted: 2811 err = sock_intr_errno(timeo); 2812 failure: 2813 *errcode = err; 2814 return NULL; 2815 } 2816 EXPORT_SYMBOL(sock_alloc_send_pskb); 2817 2818 int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg, 2819 struct sockcm_cookie *sockc) 2820 { 2821 u32 tsflags; 2822 2823 switch (cmsg->cmsg_type) { 2824 case SO_MARK: 2825 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && 2826 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) 2827 return -EPERM; 2828 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2829 return -EINVAL; 2830 sockc->mark = *(u32 *)CMSG_DATA(cmsg); 2831 break; 2832 case SO_TIMESTAMPING_OLD: 2833 case SO_TIMESTAMPING_NEW: 2834 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32))) 2835 return -EINVAL; 2836 2837 tsflags = *(u32 *)CMSG_DATA(cmsg); 2838 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK) 2839 return -EINVAL; 2840 2841 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK; 2842 sockc->tsflags |= tsflags; 2843 break; 2844 case SCM_TXTIME: 2845 if (!sock_flag(sk, SOCK_TXTIME)) 2846 return -EINVAL; 2847 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) 2848 return -EINVAL; 2849 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); 2850 break; 2851 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */ 2852 case SCM_RIGHTS: 2853 case SCM_CREDENTIALS: 2854 break; 2855 default: 2856 return -EINVAL; 2857 } 2858 return 0; 2859 } 2860 EXPORT_SYMBOL(__sock_cmsg_send); 2861 2862 int sock_cmsg_send(struct sock *sk, struct msghdr *msg, 2863 struct sockcm_cookie *sockc) 2864 { 2865 struct cmsghdr *cmsg; 2866 int ret; 2867 2868 for_each_cmsghdr(cmsg, msg) { 2869 if (!CMSG_OK(msg, cmsg)) 2870 return -EINVAL; 2871 if (cmsg->cmsg_level != SOL_SOCKET) 2872 continue; 2873 ret = __sock_cmsg_send(sk, cmsg, sockc); 2874 if (ret) 2875 return ret; 2876 } 2877 return 0; 2878 } 2879 EXPORT_SYMBOL(sock_cmsg_send); 2880 2881 static void sk_enter_memory_pressure(struct sock *sk) 2882 { 2883 if (!sk->sk_prot->enter_memory_pressure) 2884 return; 2885 2886 sk->sk_prot->enter_memory_pressure(sk); 2887 } 2888 2889 static void sk_leave_memory_pressure(struct sock *sk) 2890 { 2891 if (sk->sk_prot->leave_memory_pressure) { 2892 INDIRECT_CALL_INET_1(sk->sk_prot->leave_memory_pressure, 2893 tcp_leave_memory_pressure, sk); 2894 } else { 2895 unsigned long *memory_pressure = sk->sk_prot->memory_pressure; 2896 2897 if (memory_pressure && READ_ONCE(*memory_pressure)) 2898 WRITE_ONCE(*memory_pressure, 0); 2899 } 2900 } 2901 2902 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); 2903 2904 /** 2905 * skb_page_frag_refill - check that a page_frag contains enough room 2906 * @sz: minimum size of the fragment we want to get 2907 * @pfrag: pointer to page_frag 2908 * @gfp: priority for memory allocation 2909 * 2910 * Note: While this allocator tries to use high order pages, there is 2911 * no guarantee that allocations succeed. Therefore, @sz MUST be 2912 * less or equal than PAGE_SIZE. 2913 */ 2914 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) 2915 { 2916 if (pfrag->page) { 2917 if (page_ref_count(pfrag->page) == 1) { 2918 pfrag->offset = 0; 2919 return true; 2920 } 2921 if (pfrag->offset + sz <= pfrag->size) 2922 return true; 2923 put_page(pfrag->page); 2924 } 2925 2926 pfrag->offset = 0; 2927 if (SKB_FRAG_PAGE_ORDER && 2928 !static_branch_unlikely(&net_high_order_alloc_disable_key)) { 2929 /* Avoid direct reclaim but allow kswapd to wake */ 2930 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) | 2931 __GFP_COMP | __GFP_NOWARN | 2932 __GFP_NORETRY, 2933 SKB_FRAG_PAGE_ORDER); 2934 if (likely(pfrag->page)) { 2935 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER; 2936 return true; 2937 } 2938 } 2939 pfrag->page = alloc_page(gfp); 2940 if (likely(pfrag->page)) { 2941 pfrag->size = PAGE_SIZE; 2942 return true; 2943 } 2944 return false; 2945 } 2946 EXPORT_SYMBOL(skb_page_frag_refill); 2947 2948 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 2949 { 2950 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation))) 2951 return true; 2952 2953 sk_enter_memory_pressure(sk); 2954 sk_stream_moderate_sndbuf(sk); 2955 return false; 2956 } 2957 EXPORT_SYMBOL(sk_page_frag_refill); 2958 2959 void __lock_sock(struct sock *sk) 2960 __releases(&sk->sk_lock.slock) 2961 __acquires(&sk->sk_lock.slock) 2962 { 2963 DEFINE_WAIT(wait); 2964 2965 for (;;) { 2966 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait, 2967 TASK_UNINTERRUPTIBLE); 2968 spin_unlock_bh(&sk->sk_lock.slock); 2969 schedule(); 2970 spin_lock_bh(&sk->sk_lock.slock); 2971 if (!sock_owned_by_user(sk)) 2972 break; 2973 } 2974 finish_wait(&sk->sk_lock.wq, &wait); 2975 } 2976 2977 void __release_sock(struct sock *sk) 2978 __releases(&sk->sk_lock.slock) 2979 __acquires(&sk->sk_lock.slock) 2980 { 2981 struct sk_buff *skb, *next; 2982 2983 while ((skb = sk->sk_backlog.head) != NULL) { 2984 sk->sk_backlog.head = sk->sk_backlog.tail = NULL; 2985 2986 spin_unlock_bh(&sk->sk_lock.slock); 2987 2988 do { 2989 next = skb->next; 2990 prefetch(next); 2991 DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb)); 2992 skb_mark_not_on_list(skb); 2993 sk_backlog_rcv(sk, skb); 2994 2995 cond_resched(); 2996 2997 skb = next; 2998 } while (skb != NULL); 2999 3000 spin_lock_bh(&sk->sk_lock.slock); 3001 } 3002 3003 /* 3004 * Doing the zeroing here guarantee we can not loop forever 3005 * while a wild producer attempts to flood us. 3006 */ 3007 sk->sk_backlog.len = 0; 3008 } 3009 3010 void __sk_flush_backlog(struct sock *sk) 3011 { 3012 spin_lock_bh(&sk->sk_lock.slock); 3013 __release_sock(sk); 3014 spin_unlock_bh(&sk->sk_lock.slock); 3015 } 3016 EXPORT_SYMBOL_GPL(__sk_flush_backlog); 3017 3018 /** 3019 * sk_wait_data - wait for data to arrive at sk_receive_queue 3020 * @sk: sock to wait on 3021 * @timeo: for how long 3022 * @skb: last skb seen on sk_receive_queue 3023 * 3024 * Now socket state including sk->sk_err is changed only under lock, 3025 * hence we may omit checks after joining wait queue. 3026 * We check receive queue before schedule() only as optimization; 3027 * it is very likely that release_sock() added new data. 3028 */ 3029 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb) 3030 { 3031 DEFINE_WAIT_FUNC(wait, woken_wake_function); 3032 int rc; 3033 3034 add_wait_queue(sk_sleep(sk), &wait); 3035 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3036 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait); 3037 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 3038 remove_wait_queue(sk_sleep(sk), &wait); 3039 return rc; 3040 } 3041 EXPORT_SYMBOL(sk_wait_data); 3042 3043 /** 3044 * __sk_mem_raise_allocated - increase memory_allocated 3045 * @sk: socket 3046 * @size: memory size to allocate 3047 * @amt: pages to allocate 3048 * @kind: allocation type 3049 * 3050 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc 3051 */ 3052 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) 3053 { 3054 bool memcg_charge = mem_cgroup_sockets_enabled && sk->sk_memcg; 3055 struct proto *prot = sk->sk_prot; 3056 bool charged = true; 3057 long allocated; 3058 3059 sk_memory_allocated_add(sk, amt); 3060 allocated = sk_memory_allocated(sk); 3061 if (memcg_charge && 3062 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt, 3063 gfp_memcg_charge()))) 3064 goto suppress_allocation; 3065 3066 /* Under limit. */ 3067 if (allocated <= sk_prot_mem_limits(sk, 0)) { 3068 sk_leave_memory_pressure(sk); 3069 return 1; 3070 } 3071 3072 /* Under pressure. */ 3073 if (allocated > sk_prot_mem_limits(sk, 1)) 3074 sk_enter_memory_pressure(sk); 3075 3076 /* Over hard limit. */ 3077 if (allocated > sk_prot_mem_limits(sk, 2)) 3078 goto suppress_allocation; 3079 3080 /* guarantee minimum buffer size under pressure */ 3081 if (kind == SK_MEM_RECV) { 3082 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot)) 3083 return 1; 3084 3085 } else { /* SK_MEM_SEND */ 3086 int wmem0 = sk_get_wmem0(sk, prot); 3087 3088 if (sk->sk_type == SOCK_STREAM) { 3089 if (sk->sk_wmem_queued < wmem0) 3090 return 1; 3091 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) { 3092 return 1; 3093 } 3094 } 3095 3096 if (sk_has_memory_pressure(sk)) { 3097 u64 alloc; 3098 3099 if (!sk_under_memory_pressure(sk)) 3100 return 1; 3101 alloc = sk_sockets_allocated_read_positive(sk); 3102 if (sk_prot_mem_limits(sk, 2) > alloc * 3103 sk_mem_pages(sk->sk_wmem_queued + 3104 atomic_read(&sk->sk_rmem_alloc) + 3105 sk->sk_forward_alloc)) 3106 return 1; 3107 } 3108 3109 suppress_allocation: 3110 3111 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) { 3112 sk_stream_moderate_sndbuf(sk); 3113 3114 /* Fail only if socket is _under_ its sndbuf. 3115 * In this case we cannot block, so that we have to fail. 3116 */ 3117 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) { 3118 /* Force charge with __GFP_NOFAIL */ 3119 if (memcg_charge && !charged) { 3120 mem_cgroup_charge_skmem(sk->sk_memcg, amt, 3121 gfp_memcg_charge() | __GFP_NOFAIL); 3122 } 3123 return 1; 3124 } 3125 } 3126 3127 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged)) 3128 trace_sock_exceed_buf_limit(sk, prot, allocated, kind); 3129 3130 sk_memory_allocated_sub(sk, amt); 3131 3132 if (memcg_charge && charged) 3133 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt); 3134 3135 return 0; 3136 } 3137 3138 /** 3139 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated 3140 * @sk: socket 3141 * @size: memory size to allocate 3142 * @kind: allocation type 3143 * 3144 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means 3145 * rmem allocation. This function assumes that protocols which have 3146 * memory_pressure use sk_wmem_queued as write buffer accounting. 3147 */ 3148 int __sk_mem_schedule(struct sock *sk, int size, int kind) 3149 { 3150 int ret, amt = sk_mem_pages(size); 3151 3152 sk_forward_alloc_add(sk, amt << PAGE_SHIFT); 3153 ret = __sk_mem_raise_allocated(sk, size, amt, kind); 3154 if (!ret) 3155 sk_forward_alloc_add(sk, -(amt << PAGE_SHIFT)); 3156 return ret; 3157 } 3158 EXPORT_SYMBOL(__sk_mem_schedule); 3159 3160 /** 3161 * __sk_mem_reduce_allocated - reclaim memory_allocated 3162 * @sk: socket 3163 * @amount: number of quanta 3164 * 3165 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc 3166 */ 3167 void __sk_mem_reduce_allocated(struct sock *sk, int amount) 3168 { 3169 sk_memory_allocated_sub(sk, amount); 3170 3171 if (mem_cgroup_sockets_enabled && sk->sk_memcg) 3172 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount); 3173 3174 if (sk_under_global_memory_pressure(sk) && 3175 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0))) 3176 sk_leave_memory_pressure(sk); 3177 } 3178 3179 /** 3180 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated 3181 * @sk: socket 3182 * @amount: number of bytes (rounded down to a PAGE_SIZE multiple) 3183 */ 3184 void __sk_mem_reclaim(struct sock *sk, int amount) 3185 { 3186 amount >>= PAGE_SHIFT; 3187 sk_forward_alloc_add(sk, -(amount << PAGE_SHIFT)); 3188 __sk_mem_reduce_allocated(sk, amount); 3189 } 3190 EXPORT_SYMBOL(__sk_mem_reclaim); 3191 3192 int sk_set_peek_off(struct sock *sk, int val) 3193 { 3194 WRITE_ONCE(sk->sk_peek_off, val); 3195 return 0; 3196 } 3197 EXPORT_SYMBOL_GPL(sk_set_peek_off); 3198 3199 /* 3200 * Set of default routines for initialising struct proto_ops when 3201 * the protocol does not support a particular function. In certain 3202 * cases where it makes no sense for a protocol to have a "do nothing" 3203 * function, some default processing is provided. 3204 */ 3205 3206 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len) 3207 { 3208 return -EOPNOTSUPP; 3209 } 3210 EXPORT_SYMBOL(sock_no_bind); 3211 3212 int sock_no_connect(struct socket *sock, struct sockaddr *saddr, 3213 int len, int flags) 3214 { 3215 return -EOPNOTSUPP; 3216 } 3217 EXPORT_SYMBOL(sock_no_connect); 3218 3219 int sock_no_socketpair(struct socket *sock1, struct socket *sock2) 3220 { 3221 return -EOPNOTSUPP; 3222 } 3223 EXPORT_SYMBOL(sock_no_socketpair); 3224 3225 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, 3226 bool kern) 3227 { 3228 return -EOPNOTSUPP; 3229 } 3230 EXPORT_SYMBOL(sock_no_accept); 3231 3232 int sock_no_getname(struct socket *sock, struct sockaddr *saddr, 3233 int peer) 3234 { 3235 return -EOPNOTSUPP; 3236 } 3237 EXPORT_SYMBOL(sock_no_getname); 3238 3239 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) 3240 { 3241 return -EOPNOTSUPP; 3242 } 3243 EXPORT_SYMBOL(sock_no_ioctl); 3244 3245 int sock_no_listen(struct socket *sock, int backlog) 3246 { 3247 return -EOPNOTSUPP; 3248 } 3249 EXPORT_SYMBOL(sock_no_listen); 3250 3251 int sock_no_shutdown(struct socket *sock, int how) 3252 { 3253 return -EOPNOTSUPP; 3254 } 3255 EXPORT_SYMBOL(sock_no_shutdown); 3256 3257 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len) 3258 { 3259 return -EOPNOTSUPP; 3260 } 3261 EXPORT_SYMBOL(sock_no_sendmsg); 3262 3263 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len) 3264 { 3265 return -EOPNOTSUPP; 3266 } 3267 EXPORT_SYMBOL(sock_no_sendmsg_locked); 3268 3269 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len, 3270 int flags) 3271 { 3272 return -EOPNOTSUPP; 3273 } 3274 EXPORT_SYMBOL(sock_no_recvmsg); 3275 3276 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma) 3277 { 3278 /* Mirror missing mmap method error code */ 3279 return -ENODEV; 3280 } 3281 EXPORT_SYMBOL(sock_no_mmap); 3282 3283 /* 3284 * When a file is received (via SCM_RIGHTS, etc), we must bump the 3285 * various sock-based usage counts. 3286 */ 3287 void __receive_sock(struct file *file) 3288 { 3289 struct socket *sock; 3290 3291 sock = sock_from_file(file); 3292 if (sock) { 3293 sock_update_netprioidx(&sock->sk->sk_cgrp_data); 3294 sock_update_classid(&sock->sk->sk_cgrp_data); 3295 } 3296 } 3297 3298 /* 3299 * Default Socket Callbacks 3300 */ 3301 3302 static void sock_def_wakeup(struct sock *sk) 3303 { 3304 struct socket_wq *wq; 3305 3306 rcu_read_lock(); 3307 wq = rcu_dereference(sk->sk_wq); 3308 if (skwq_has_sleeper(wq)) 3309 wake_up_interruptible_all(&wq->wait); 3310 rcu_read_unlock(); 3311 } 3312 3313 static void sock_def_error_report(struct sock *sk) 3314 { 3315 struct socket_wq *wq; 3316 3317 rcu_read_lock(); 3318 wq = rcu_dereference(sk->sk_wq); 3319 if (skwq_has_sleeper(wq)) 3320 wake_up_interruptible_poll(&wq->wait, EPOLLERR); 3321 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR); 3322 rcu_read_unlock(); 3323 } 3324 3325 void sock_def_readable(struct sock *sk) 3326 { 3327 struct socket_wq *wq; 3328 3329 trace_sk_data_ready(sk); 3330 3331 rcu_read_lock(); 3332 wq = rcu_dereference(sk->sk_wq); 3333 if (skwq_has_sleeper(wq)) 3334 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI | 3335 EPOLLRDNORM | EPOLLRDBAND); 3336 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 3337 rcu_read_unlock(); 3338 } 3339 3340 static void sock_def_write_space(struct sock *sk) 3341 { 3342 struct socket_wq *wq; 3343 3344 rcu_read_lock(); 3345 3346 /* Do not wake up a writer until he can make "significant" 3347 * progress. --DaveM 3348 */ 3349 if (sock_writeable(sk)) { 3350 wq = rcu_dereference(sk->sk_wq); 3351 if (skwq_has_sleeper(wq)) 3352 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3353 EPOLLWRNORM | EPOLLWRBAND); 3354 3355 /* Should agree with poll, otherwise some programs break */ 3356 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 3357 } 3358 3359 rcu_read_unlock(); 3360 } 3361 3362 /* An optimised version of sock_def_write_space(), should only be called 3363 * for SOCK_RCU_FREE sockets under RCU read section and after putting 3364 * ->sk_wmem_alloc. 3365 */ 3366 static void sock_def_write_space_wfree(struct sock *sk) 3367 { 3368 /* Do not wake up a writer until he can make "significant" 3369 * progress. --DaveM 3370 */ 3371 if (sock_writeable(sk)) { 3372 struct socket_wq *wq = rcu_dereference(sk->sk_wq); 3373 3374 /* rely on refcount_sub from sock_wfree() */ 3375 smp_mb__after_atomic(); 3376 if (wq && waitqueue_active(&wq->wait)) 3377 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 3378 EPOLLWRNORM | EPOLLWRBAND); 3379 3380 /* Should agree with poll, otherwise some programs break */ 3381 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 3382 } 3383 } 3384 3385 static void sock_def_destruct(struct sock *sk) 3386 { 3387 } 3388 3389 void sk_send_sigurg(struct sock *sk) 3390 { 3391 if (sk->sk_socket && sk->sk_socket->file) 3392 if (send_sigurg(&sk->sk_socket->file->f_owner)) 3393 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI); 3394 } 3395 EXPORT_SYMBOL(sk_send_sigurg); 3396 3397 void sk_reset_timer(struct sock *sk, struct timer_list* timer, 3398 unsigned long expires) 3399 { 3400 if (!mod_timer(timer, expires)) 3401 sock_hold(sk); 3402 } 3403 EXPORT_SYMBOL(sk_reset_timer); 3404 3405 void sk_stop_timer(struct sock *sk, struct timer_list* timer) 3406 { 3407 if (del_timer(timer)) 3408 __sock_put(sk); 3409 } 3410 EXPORT_SYMBOL(sk_stop_timer); 3411 3412 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer) 3413 { 3414 if (del_timer_sync(timer)) 3415 __sock_put(sk); 3416 } 3417 EXPORT_SYMBOL(sk_stop_timer_sync); 3418 3419 void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid) 3420 { 3421 sk_init_common(sk); 3422 sk->sk_send_head = NULL; 3423 3424 timer_setup(&sk->sk_timer, NULL, 0); 3425 3426 sk->sk_allocation = GFP_KERNEL; 3427 sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default); 3428 sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); 3429 sk->sk_state = TCP_CLOSE; 3430 sk->sk_use_task_frag = true; 3431 sk_set_socket(sk, sock); 3432 3433 sock_set_flag(sk, SOCK_ZAPPED); 3434 3435 if (sock) { 3436 sk->sk_type = sock->type; 3437 RCU_INIT_POINTER(sk->sk_wq, &sock->wq); 3438 sock->sk = sk; 3439 } else { 3440 RCU_INIT_POINTER(sk->sk_wq, NULL); 3441 } 3442 sk->sk_uid = uid; 3443 3444 rwlock_init(&sk->sk_callback_lock); 3445 if (sk->sk_kern_sock) 3446 lockdep_set_class_and_name( 3447 &sk->sk_callback_lock, 3448 af_kern_callback_keys + sk->sk_family, 3449 af_family_kern_clock_key_strings[sk->sk_family]); 3450 else 3451 lockdep_set_class_and_name( 3452 &sk->sk_callback_lock, 3453 af_callback_keys + sk->sk_family, 3454 af_family_clock_key_strings[sk->sk_family]); 3455 3456 sk->sk_state_change = sock_def_wakeup; 3457 sk->sk_data_ready = sock_def_readable; 3458 sk->sk_write_space = sock_def_write_space; 3459 sk->sk_error_report = sock_def_error_report; 3460 sk->sk_destruct = sock_def_destruct; 3461 3462 sk->sk_frag.page = NULL; 3463 sk->sk_frag.offset = 0; 3464 sk->sk_peek_off = -1; 3465 3466 sk->sk_peer_pid = NULL; 3467 sk->sk_peer_cred = NULL; 3468 spin_lock_init(&sk->sk_peer_lock); 3469 3470 sk->sk_write_pending = 0; 3471 sk->sk_rcvlowat = 1; 3472 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT; 3473 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT; 3474 3475 sk->sk_stamp = SK_DEFAULT_STAMP; 3476 #if BITS_PER_LONG==32 3477 seqlock_init(&sk->sk_stamp_seq); 3478 #endif 3479 atomic_set(&sk->sk_zckey, 0); 3480 3481 #ifdef CONFIG_NET_RX_BUSY_POLL 3482 sk->sk_napi_id = 0; 3483 sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read); 3484 #endif 3485 3486 sk->sk_max_pacing_rate = ~0UL; 3487 sk->sk_pacing_rate = ~0UL; 3488 WRITE_ONCE(sk->sk_pacing_shift, 10); 3489 sk->sk_incoming_cpu = -1; 3490 3491 sk_rx_queue_clear(sk); 3492 /* 3493 * Before updating sk_refcnt, we must commit prior changes to memory 3494 * (Documentation/RCU/rculist_nulls.rst for details) 3495 */ 3496 smp_wmb(); 3497 refcount_set(&sk->sk_refcnt, 1); 3498 atomic_set(&sk->sk_drops, 0); 3499 } 3500 EXPORT_SYMBOL(sock_init_data_uid); 3501 3502 void sock_init_data(struct socket *sock, struct sock *sk) 3503 { 3504 kuid_t uid = sock ? 3505 SOCK_INODE(sock)->i_uid : 3506 make_kuid(sock_net(sk)->user_ns, 0); 3507 3508 sock_init_data_uid(sock, sk, uid); 3509 } 3510 EXPORT_SYMBOL(sock_init_data); 3511 3512 void lock_sock_nested(struct sock *sk, int subclass) 3513 { 3514 /* The sk_lock has mutex_lock() semantics here. */ 3515 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_); 3516 3517 might_sleep(); 3518 spin_lock_bh(&sk->sk_lock.slock); 3519 if (sock_owned_by_user_nocheck(sk)) 3520 __lock_sock(sk); 3521 sk->sk_lock.owned = 1; 3522 spin_unlock_bh(&sk->sk_lock.slock); 3523 } 3524 EXPORT_SYMBOL(lock_sock_nested); 3525 3526 void release_sock(struct sock *sk) 3527 { 3528 spin_lock_bh(&sk->sk_lock.slock); 3529 if (sk->sk_backlog.tail) 3530 __release_sock(sk); 3531 3532 /* Warning : release_cb() might need to release sk ownership, 3533 * ie call sock_release_ownership(sk) before us. 3534 */ 3535 if (sk->sk_prot->release_cb) 3536 sk->sk_prot->release_cb(sk); 3537 3538 sock_release_ownership(sk); 3539 if (waitqueue_active(&sk->sk_lock.wq)) 3540 wake_up(&sk->sk_lock.wq); 3541 spin_unlock_bh(&sk->sk_lock.slock); 3542 } 3543 EXPORT_SYMBOL(release_sock); 3544 3545 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock) 3546 { 3547 might_sleep(); 3548 spin_lock_bh(&sk->sk_lock.slock); 3549 3550 if (!sock_owned_by_user_nocheck(sk)) { 3551 /* 3552 * Fast path return with bottom halves disabled and 3553 * sock::sk_lock.slock held. 3554 * 3555 * The 'mutex' is not contended and holding 3556 * sock::sk_lock.slock prevents all other lockers to 3557 * proceed so the corresponding unlock_sock_fast() can 3558 * avoid the slow path of release_sock() completely and 3559 * just release slock. 3560 * 3561 * From a semantical POV this is equivalent to 'acquiring' 3562 * the 'mutex', hence the corresponding lockdep 3563 * mutex_release() has to happen in the fast path of 3564 * unlock_sock_fast(). 3565 */ 3566 return false; 3567 } 3568 3569 __lock_sock(sk); 3570 sk->sk_lock.owned = 1; 3571 __acquire(&sk->sk_lock.slock); 3572 spin_unlock_bh(&sk->sk_lock.slock); 3573 return true; 3574 } 3575 EXPORT_SYMBOL(__lock_sock_fast); 3576 3577 int sock_gettstamp(struct socket *sock, void __user *userstamp, 3578 bool timeval, bool time32) 3579 { 3580 struct sock *sk = sock->sk; 3581 struct timespec64 ts; 3582 3583 sock_enable_timestamp(sk, SOCK_TIMESTAMP); 3584 ts = ktime_to_timespec64(sock_read_timestamp(sk)); 3585 if (ts.tv_sec == -1) 3586 return -ENOENT; 3587 if (ts.tv_sec == 0) { 3588 ktime_t kt = ktime_get_real(); 3589 sock_write_timestamp(sk, kt); 3590 ts = ktime_to_timespec64(kt); 3591 } 3592 3593 if (timeval) 3594 ts.tv_nsec /= 1000; 3595 3596 #ifdef CONFIG_COMPAT_32BIT_TIME 3597 if (time32) 3598 return put_old_timespec32(&ts, userstamp); 3599 #endif 3600 #ifdef CONFIG_SPARC64 3601 /* beware of padding in sparc64 timeval */ 3602 if (timeval && !in_compat_syscall()) { 3603 struct __kernel_old_timeval __user tv = { 3604 .tv_sec = ts.tv_sec, 3605 .tv_usec = ts.tv_nsec, 3606 }; 3607 if (copy_to_user(userstamp, &tv, sizeof(tv))) 3608 return -EFAULT; 3609 return 0; 3610 } 3611 #endif 3612 return put_timespec64(&ts, userstamp); 3613 } 3614 EXPORT_SYMBOL(sock_gettstamp); 3615 3616 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag) 3617 { 3618 if (!sock_flag(sk, flag)) { 3619 unsigned long previous_flags = sk->sk_flags; 3620 3621 sock_set_flag(sk, flag); 3622 /* 3623 * we just set one of the two flags which require net 3624 * time stamping, but time stamping might have been on 3625 * already because of the other one 3626 */ 3627 if (sock_needs_netstamp(sk) && 3628 !(previous_flags & SK_FLAGS_TIMESTAMP)) 3629 net_enable_timestamp(); 3630 } 3631 } 3632 3633 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, 3634 int level, int type) 3635 { 3636 struct sock_exterr_skb *serr; 3637 struct sk_buff *skb; 3638 int copied, err; 3639 3640 err = -EAGAIN; 3641 skb = sock_dequeue_err_skb(sk); 3642 if (skb == NULL) 3643 goto out; 3644 3645 copied = skb->len; 3646 if (copied > len) { 3647 msg->msg_flags |= MSG_TRUNC; 3648 copied = len; 3649 } 3650 err = skb_copy_datagram_msg(skb, 0, msg, copied); 3651 if (err) 3652 goto out_free_skb; 3653 3654 sock_recv_timestamp(msg, sk, skb); 3655 3656 serr = SKB_EXT_ERR(skb); 3657 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee); 3658 3659 msg->msg_flags |= MSG_ERRQUEUE; 3660 err = copied; 3661 3662 out_free_skb: 3663 kfree_skb(skb); 3664 out: 3665 return err; 3666 } 3667 EXPORT_SYMBOL(sock_recv_errqueue); 3668 3669 /* 3670 * Get a socket option on an socket. 3671 * 3672 * FIX: POSIX 1003.1g is very ambiguous here. It states that 3673 * asynchronous errors should be reported by getsockopt. We assume 3674 * this means if you specify SO_ERROR (otherwise whats the point of it). 3675 */ 3676 int sock_common_getsockopt(struct socket *sock, int level, int optname, 3677 char __user *optval, int __user *optlen) 3678 { 3679 struct sock *sk = sock->sk; 3680 3681 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 3682 return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen); 3683 } 3684 EXPORT_SYMBOL(sock_common_getsockopt); 3685 3686 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, 3687 int flags) 3688 { 3689 struct sock *sk = sock->sk; 3690 int addr_len = 0; 3691 int err; 3692 3693 err = sk->sk_prot->recvmsg(sk, msg, size, flags, &addr_len); 3694 if (err >= 0) 3695 msg->msg_namelen = addr_len; 3696 return err; 3697 } 3698 EXPORT_SYMBOL(sock_common_recvmsg); 3699 3700 /* 3701 * Set socket options on an inet socket. 3702 */ 3703 int sock_common_setsockopt(struct socket *sock, int level, int optname, 3704 sockptr_t optval, unsigned int optlen) 3705 { 3706 struct sock *sk = sock->sk; 3707 3708 /* IPV6_ADDRFORM can change sk->sk_prot under us. */ 3709 return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen); 3710 } 3711 EXPORT_SYMBOL(sock_common_setsockopt); 3712 3713 void sk_common_release(struct sock *sk) 3714 { 3715 if (sk->sk_prot->destroy) 3716 sk->sk_prot->destroy(sk); 3717 3718 /* 3719 * Observation: when sk_common_release is called, processes have 3720 * no access to socket. But net still has. 3721 * Step one, detach it from networking: 3722 * 3723 * A. Remove from hash tables. 3724 */ 3725 3726 sk->sk_prot->unhash(sk); 3727 3728 /* 3729 * In this point socket cannot receive new packets, but it is possible 3730 * that some packets are in flight because some CPU runs receiver and 3731 * did hash table lookup before we unhashed socket. They will achieve 3732 * receive queue and will be purged by socket destructor. 3733 * 3734 * Also we still have packets pending on receive queue and probably, 3735 * our own packets waiting in device queues. sock_destroy will drain 3736 * receive queue, but transmitted packets will delay socket destruction 3737 * until the last reference will be released. 3738 */ 3739 3740 sock_orphan(sk); 3741 3742 xfrm_sk_free_policy(sk); 3743 3744 sock_put(sk); 3745 } 3746 EXPORT_SYMBOL(sk_common_release); 3747 3748 void sk_get_meminfo(const struct sock *sk, u32 *mem) 3749 { 3750 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS); 3751 3752 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); 3753 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); 3754 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); 3755 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf); 3756 mem[SK_MEMINFO_FWD_ALLOC] = sk_forward_alloc_get(sk); 3757 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued); 3758 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); 3759 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len); 3760 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops); 3761 } 3762 3763 #ifdef CONFIG_PROC_FS 3764 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); 3765 3766 int sock_prot_inuse_get(struct net *net, struct proto *prot) 3767 { 3768 int cpu, idx = prot->inuse_idx; 3769 int res = 0; 3770 3771 for_each_possible_cpu(cpu) 3772 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx]; 3773 3774 return res >= 0 ? res : 0; 3775 } 3776 EXPORT_SYMBOL_GPL(sock_prot_inuse_get); 3777 3778 int sock_inuse_get(struct net *net) 3779 { 3780 int cpu, res = 0; 3781 3782 for_each_possible_cpu(cpu) 3783 res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; 3784 3785 return res; 3786 } 3787 3788 EXPORT_SYMBOL_GPL(sock_inuse_get); 3789 3790 static int __net_init sock_inuse_init_net(struct net *net) 3791 { 3792 net->core.prot_inuse = alloc_percpu(struct prot_inuse); 3793 if (net->core.prot_inuse == NULL) 3794 return -ENOMEM; 3795 return 0; 3796 } 3797 3798 static void __net_exit sock_inuse_exit_net(struct net *net) 3799 { 3800 free_percpu(net->core.prot_inuse); 3801 } 3802 3803 static struct pernet_operations net_inuse_ops = { 3804 .init = sock_inuse_init_net, 3805 .exit = sock_inuse_exit_net, 3806 }; 3807 3808 static __init int net_inuse_init(void) 3809 { 3810 if (register_pernet_subsys(&net_inuse_ops)) 3811 panic("Cannot initialize net inuse counters"); 3812 3813 return 0; 3814 } 3815 3816 core_initcall(net_inuse_init); 3817 3818 static int assign_proto_idx(struct proto *prot) 3819 { 3820 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR); 3821 3822 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) { 3823 pr_err("PROTO_INUSE_NR exhausted\n"); 3824 return -ENOSPC; 3825 } 3826 3827 set_bit(prot->inuse_idx, proto_inuse_idx); 3828 return 0; 3829 } 3830 3831 static void release_proto_idx(struct proto *prot) 3832 { 3833 if (prot->inuse_idx != PROTO_INUSE_NR - 1) 3834 clear_bit(prot->inuse_idx, proto_inuse_idx); 3835 } 3836 #else 3837 static inline int assign_proto_idx(struct proto *prot) 3838 { 3839 return 0; 3840 } 3841 3842 static inline void release_proto_idx(struct proto *prot) 3843 { 3844 } 3845 3846 #endif 3847 3848 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) 3849 { 3850 if (!twsk_prot) 3851 return; 3852 kfree(twsk_prot->twsk_slab_name); 3853 twsk_prot->twsk_slab_name = NULL; 3854 kmem_cache_destroy(twsk_prot->twsk_slab); 3855 twsk_prot->twsk_slab = NULL; 3856 } 3857 3858 static int tw_prot_init(const struct proto *prot) 3859 { 3860 struct timewait_sock_ops *twsk_prot = prot->twsk_prot; 3861 3862 if (!twsk_prot) 3863 return 0; 3864 3865 twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", 3866 prot->name); 3867 if (!twsk_prot->twsk_slab_name) 3868 return -ENOMEM; 3869 3870 twsk_prot->twsk_slab = 3871 kmem_cache_create(twsk_prot->twsk_slab_name, 3872 twsk_prot->twsk_obj_size, 0, 3873 SLAB_ACCOUNT | prot->slab_flags, 3874 NULL); 3875 if (!twsk_prot->twsk_slab) { 3876 pr_crit("%s: Can't create timewait sock SLAB cache!\n", 3877 prot->name); 3878 return -ENOMEM; 3879 } 3880 3881 return 0; 3882 } 3883 3884 static void req_prot_cleanup(struct request_sock_ops *rsk_prot) 3885 { 3886 if (!rsk_prot) 3887 return; 3888 kfree(rsk_prot->slab_name); 3889 rsk_prot->slab_name = NULL; 3890 kmem_cache_destroy(rsk_prot->slab); 3891 rsk_prot->slab = NULL; 3892 } 3893 3894 static int req_prot_init(const struct proto *prot) 3895 { 3896 struct request_sock_ops *rsk_prot = prot->rsk_prot; 3897 3898 if (!rsk_prot) 3899 return 0; 3900 3901 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s", 3902 prot->name); 3903 if (!rsk_prot->slab_name) 3904 return -ENOMEM; 3905 3906 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name, 3907 rsk_prot->obj_size, 0, 3908 SLAB_ACCOUNT | prot->slab_flags, 3909 NULL); 3910 3911 if (!rsk_prot->slab) { 3912 pr_crit("%s: Can't create request sock SLAB cache!\n", 3913 prot->name); 3914 return -ENOMEM; 3915 } 3916 return 0; 3917 } 3918 3919 int proto_register(struct proto *prot, int alloc_slab) 3920 { 3921 int ret = -ENOBUFS; 3922 3923 if (prot->memory_allocated && !prot->sysctl_mem) { 3924 pr_err("%s: missing sysctl_mem\n", prot->name); 3925 return -EINVAL; 3926 } 3927 if (prot->memory_allocated && !prot->per_cpu_fw_alloc) { 3928 pr_err("%s: missing per_cpu_fw_alloc\n", prot->name); 3929 return -EINVAL; 3930 } 3931 if (alloc_slab) { 3932 prot->slab = kmem_cache_create_usercopy(prot->name, 3933 prot->obj_size, 0, 3934 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT | 3935 prot->slab_flags, 3936 prot->useroffset, prot->usersize, 3937 NULL); 3938 3939 if (prot->slab == NULL) { 3940 pr_crit("%s: Can't create sock SLAB cache!\n", 3941 prot->name); 3942 goto out; 3943 } 3944 3945 if (req_prot_init(prot)) 3946 goto out_free_request_sock_slab; 3947 3948 if (tw_prot_init(prot)) 3949 goto out_free_timewait_sock_slab; 3950 } 3951 3952 mutex_lock(&proto_list_mutex); 3953 ret = assign_proto_idx(prot); 3954 if (ret) { 3955 mutex_unlock(&proto_list_mutex); 3956 goto out_free_timewait_sock_slab; 3957 } 3958 list_add(&prot->node, &proto_list); 3959 mutex_unlock(&proto_list_mutex); 3960 return ret; 3961 3962 out_free_timewait_sock_slab: 3963 if (alloc_slab) 3964 tw_prot_cleanup(prot->twsk_prot); 3965 out_free_request_sock_slab: 3966 if (alloc_slab) { 3967 req_prot_cleanup(prot->rsk_prot); 3968 3969 kmem_cache_destroy(prot->slab); 3970 prot->slab = NULL; 3971 } 3972 out: 3973 return ret; 3974 } 3975 EXPORT_SYMBOL(proto_register); 3976 3977 void proto_unregister(struct proto *prot) 3978 { 3979 mutex_lock(&proto_list_mutex); 3980 release_proto_idx(prot); 3981 list_del(&prot->node); 3982 mutex_unlock(&proto_list_mutex); 3983 3984 kmem_cache_destroy(prot->slab); 3985 prot->slab = NULL; 3986 3987 req_prot_cleanup(prot->rsk_prot); 3988 tw_prot_cleanup(prot->twsk_prot); 3989 } 3990 EXPORT_SYMBOL(proto_unregister); 3991 3992 int sock_load_diag_module(int family, int protocol) 3993 { 3994 if (!protocol) { 3995 if (!sock_is_registered(family)) 3996 return -ENOENT; 3997 3998 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK, 3999 NETLINK_SOCK_DIAG, family); 4000 } 4001 4002 #ifdef CONFIG_INET 4003 if (family == AF_INET && 4004 protocol != IPPROTO_RAW && 4005 protocol < MAX_INET_PROTOS && 4006 !rcu_access_pointer(inet_protos[protocol])) 4007 return -ENOENT; 4008 #endif 4009 4010 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK, 4011 NETLINK_SOCK_DIAG, family, protocol); 4012 } 4013 EXPORT_SYMBOL(sock_load_diag_module); 4014 4015 #ifdef CONFIG_PROC_FS 4016 static void *proto_seq_start(struct seq_file *seq, loff_t *pos) 4017 __acquires(proto_list_mutex) 4018 { 4019 mutex_lock(&proto_list_mutex); 4020 return seq_list_start_head(&proto_list, *pos); 4021 } 4022 4023 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4024 { 4025 return seq_list_next(v, &proto_list, pos); 4026 } 4027 4028 static void proto_seq_stop(struct seq_file *seq, void *v) 4029 __releases(proto_list_mutex) 4030 { 4031 mutex_unlock(&proto_list_mutex); 4032 } 4033 4034 static char proto_method_implemented(const void *method) 4035 { 4036 return method == NULL ? 'n' : 'y'; 4037 } 4038 static long sock_prot_memory_allocated(struct proto *proto) 4039 { 4040 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L; 4041 } 4042 4043 static const char *sock_prot_memory_pressure(struct proto *proto) 4044 { 4045 return proto->memory_pressure != NULL ? 4046 proto_memory_pressure(proto) ? "yes" : "no" : "NI"; 4047 } 4048 4049 static void proto_seq_printf(struct seq_file *seq, struct proto *proto) 4050 { 4051 4052 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s " 4053 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", 4054 proto->name, 4055 proto->obj_size, 4056 sock_prot_inuse_get(seq_file_net(seq), proto), 4057 sock_prot_memory_allocated(proto), 4058 sock_prot_memory_pressure(proto), 4059 proto->max_header, 4060 proto->slab == NULL ? "no" : "yes", 4061 module_name(proto->owner), 4062 proto_method_implemented(proto->close), 4063 proto_method_implemented(proto->connect), 4064 proto_method_implemented(proto->disconnect), 4065 proto_method_implemented(proto->accept), 4066 proto_method_implemented(proto->ioctl), 4067 proto_method_implemented(proto->init), 4068 proto_method_implemented(proto->destroy), 4069 proto_method_implemented(proto->shutdown), 4070 proto_method_implemented(proto->setsockopt), 4071 proto_method_implemented(proto->getsockopt), 4072 proto_method_implemented(proto->sendmsg), 4073 proto_method_implemented(proto->recvmsg), 4074 proto_method_implemented(proto->bind), 4075 proto_method_implemented(proto->backlog_rcv), 4076 proto_method_implemented(proto->hash), 4077 proto_method_implemented(proto->unhash), 4078 proto_method_implemented(proto->get_port), 4079 proto_method_implemented(proto->enter_memory_pressure)); 4080 } 4081 4082 static int proto_seq_show(struct seq_file *seq, void *v) 4083 { 4084 if (v == &proto_list) 4085 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s", 4086 "protocol", 4087 "size", 4088 "sockets", 4089 "memory", 4090 "press", 4091 "maxhdr", 4092 "slab", 4093 "module", 4094 "cl co di ac io in de sh ss gs se re bi br ha uh gp em\n"); 4095 else 4096 proto_seq_printf(seq, list_entry(v, struct proto, node)); 4097 return 0; 4098 } 4099 4100 static const struct seq_operations proto_seq_ops = { 4101 .start = proto_seq_start, 4102 .next = proto_seq_next, 4103 .stop = proto_seq_stop, 4104 .show = proto_seq_show, 4105 }; 4106 4107 static __net_init int proto_init_net(struct net *net) 4108 { 4109 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops, 4110 sizeof(struct seq_net_private))) 4111 return -ENOMEM; 4112 4113 return 0; 4114 } 4115 4116 static __net_exit void proto_exit_net(struct net *net) 4117 { 4118 remove_proc_entry("protocols", net->proc_net); 4119 } 4120 4121 4122 static __net_initdata struct pernet_operations proto_net_ops = { 4123 .init = proto_init_net, 4124 .exit = proto_exit_net, 4125 }; 4126 4127 static int __init proto_init(void) 4128 { 4129 return register_pernet_subsys(&proto_net_ops); 4130 } 4131 4132 subsys_initcall(proto_init); 4133 4134 #endif /* PROC_FS */ 4135 4136 #ifdef CONFIG_NET_RX_BUSY_POLL 4137 bool sk_busy_loop_end(void *p, unsigned long start_time) 4138 { 4139 struct sock *sk = p; 4140 4141 if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) 4142 return true; 4143 4144 if (sk_is_udp(sk) && 4145 !skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) 4146 return true; 4147 4148 return sk_busy_loop_timeout(sk, start_time); 4149 } 4150 EXPORT_SYMBOL(sk_busy_loop_end); 4151 #endif /* CONFIG_NET_RX_BUSY_POLL */ 4152 4153 int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len) 4154 { 4155 if (!sk->sk_prot->bind_add) 4156 return -EOPNOTSUPP; 4157 return sk->sk_prot->bind_add(sk, addr, addr_len); 4158 } 4159 EXPORT_SYMBOL(sock_bind_add); 4160 4161 /* Copy 'size' bytes from userspace and return `size` back to userspace */ 4162 int sock_ioctl_inout(struct sock *sk, unsigned int cmd, 4163 void __user *arg, void *karg, size_t size) 4164 { 4165 int ret; 4166 4167 if (copy_from_user(karg, arg, size)) 4168 return -EFAULT; 4169 4170 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, karg); 4171 if (ret) 4172 return ret; 4173 4174 if (copy_to_user(arg, karg, size)) 4175 return -EFAULT; 4176 4177 return 0; 4178 } 4179 EXPORT_SYMBOL(sock_ioctl_inout); 4180 4181 /* This is the most common ioctl prep function, where the result (4 bytes) is 4182 * copied back to userspace if the ioctl() returns successfully. No input is 4183 * copied from userspace as input argument. 4184 */ 4185 static int sock_ioctl_out(struct sock *sk, unsigned int cmd, void __user *arg) 4186 { 4187 int ret, karg = 0; 4188 4189 ret = READ_ONCE(sk->sk_prot)->ioctl(sk, cmd, &karg); 4190 if (ret) 4191 return ret; 4192 4193 return put_user(karg, (int __user *)arg); 4194 } 4195 4196 /* A wrapper around sock ioctls, which copies the data from userspace 4197 * (depending on the protocol/ioctl), and copies back the result to userspace. 4198 * The main motivation for this function is to pass kernel memory to the 4199 * protocol ioctl callbacks, instead of userspace memory. 4200 */ 4201 int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg) 4202 { 4203 int rc = 1; 4204 4205 if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET) 4206 rc = ipmr_sk_ioctl(sk, cmd, arg); 4207 else if (sk->sk_type == SOCK_RAW && sk->sk_family == AF_INET6) 4208 rc = ip6mr_sk_ioctl(sk, cmd, arg); 4209 else if (sk_is_phonet(sk)) 4210 rc = phonet_sk_ioctl(sk, cmd, arg); 4211 4212 /* If ioctl was processed, returns its value */ 4213 if (rc <= 0) 4214 return rc; 4215 4216 /* Otherwise call the default handler */ 4217 return sock_ioctl_out(sk, cmd, arg); 4218 } 4219 EXPORT_SYMBOL(sk_ioctl); 4220