tcp.c (db6da59cf27b5661ced03754ae0550f8914eda9e) tcp.c (e1d001fa5b477c4da46a29be1fcece91db7c7c6f)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Implementation of the Transmission Control Protocol(TCP).
8 *

--- 585 unchanged lines hidden (view full) ---

594 if (READ_ONCE(sk->sk_err) ||
595 !skb_queue_empty_lockless(&sk->sk_error_queue))
596 mask |= EPOLLERR;
597
598 return mask;
599}
600EXPORT_SYMBOL(tcp_poll);
601
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Implementation of the Transmission Control Protocol(TCP).
8 *

--- 585 unchanged lines hidden (view full) ---

594 if (READ_ONCE(sk->sk_err) ||
595 !skb_queue_empty_lockless(&sk->sk_error_queue))
596 mask |= EPOLLERR;
597
598 return mask;
599}
600EXPORT_SYMBOL(tcp_poll);
601
602int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
602int tcp_ioctl(struct sock *sk, int cmd, int *karg)
603{
604 struct tcp_sock *tp = tcp_sk(sk);
605 int answ;
606 bool slow;
607
608 switch (cmd) {
609 case SIOCINQ:
610 if (sk->sk_state == TCP_LISTEN)

--- 25 unchanged lines hidden (view full) ---

636 else
637 answ = READ_ONCE(tp->write_seq) -
638 READ_ONCE(tp->snd_nxt);
639 break;
640 default:
641 return -ENOIOCTLCMD;
642 }
643
603{
604 struct tcp_sock *tp = tcp_sk(sk);
605 int answ;
606 bool slow;
607
608 switch (cmd) {
609 case SIOCINQ:
610 if (sk->sk_state == TCP_LISTEN)

--- 25 unchanged lines hidden (view full) ---

636 else
637 answ = READ_ONCE(tp->write_seq) -
638 READ_ONCE(tp->snd_nxt);
639 break;
640 default:
641 return -ENOIOCTLCMD;
642 }
643
644 return put_user(answ, (int __user *)arg);
644 *karg = answ;
645 return 0;
645}
646EXPORT_SYMBOL(tcp_ioctl);
647
648void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
649{
650 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
651 tp->pushed_seq = tp->write_seq;
652}

--- 200 unchanged lines hidden (view full) ---

853
854 if (spliced)
855 return spliced;
856
857 return ret;
858}
859EXPORT_SYMBOL(tcp_splice_read);
860
646}
647EXPORT_SYMBOL(tcp_ioctl);
648
649void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
650{
651 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
652 tp->pushed_seq = tp->write_seq;
653}

--- 200 unchanged lines hidden (view full) ---

854
855 if (spliced)
856 return spliced;
857
858 return ret;
859}
860EXPORT_SYMBOL(tcp_splice_read);
861
861struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
862struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
862 bool force_schedule)
863{
864 struct sk_buff *skb;
865
863 bool force_schedule)
864{
865 struct sk_buff *skb;
866
866 skb = alloc_skb_fclone(size + MAX_TCP_HEADER, gfp);
867 skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
867 if (likely(skb)) {
868 bool mem_scheduled;
869
870 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
871 if (force_schedule) {
872 mem_scheduled = true;
873 sk_forced_mem_schedule(sk, skb->truesize);
874 } else {

--- 77 unchanged lines hidden (view full) ---

952
953 sk_mem_charge(sk, extra);
954 skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY;
955 }
956 return 0;
957}
958
959
868 if (likely(skb)) {
869 bool mem_scheduled;
870
871 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
872 if (force_schedule) {
873 mem_scheduled = true;
874 sk_forced_mem_schedule(sk, skb->truesize);
875 } else {

--- 77 unchanged lines hidden (view full) ---

953
954 sk_mem_charge(sk, extra);
955 skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY;
956 }
957 return 0;
958}
959
960
960static int tcp_wmem_schedule(struct sock *sk, int copy)
961int tcp_wmem_schedule(struct sock *sk, int copy)
961{
962 int left;
963
964 if (likely(sk_wmem_schedule(sk, copy)))
965 return copy;
966
967 /* We could be in trouble if we have nothing queued.
968 * Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0]
969 * to guarantee some progress.
970 */
971 left = sock_net(sk)->ipv4.sysctl_tcp_wmem[0] - sk->sk_wmem_queued;
972 if (left > 0)
973 sk_forced_mem_schedule(sk, min(left, copy));
974 return min(copy, sk->sk_forward_alloc);
975}
976
962{
963 int left;
964
965 if (likely(sk_wmem_schedule(sk, copy)))
966 return copy;
967
968 /* We could be in trouble if we have nothing queued.
969 * Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0]
970 * to guarantee some progress.
971 */
972 left = sock_net(sk)->ipv4.sysctl_tcp_wmem[0] - sk->sk_wmem_queued;
973 if (left > 0)
974 sk_forced_mem_schedule(sk, min(left, copy));
975 return min(copy, sk->sk_forward_alloc);
976}
977
977static struct sk_buff *tcp_build_frag(struct sock *sk, int size_goal, int flags,
978 struct page *page, int offset, size_t *size)
979{
980 struct sk_buff *skb = tcp_write_queue_tail(sk);
981 struct tcp_sock *tp = tcp_sk(sk);
982 bool can_coalesce;
983 int copy, i;
984
985 if (!skb || (copy = size_goal - skb->len) <= 0 ||
986 !tcp_skb_can_collapse_to(skb)) {
987new_segment:
988 if (!sk_stream_memory_free(sk))
989 return NULL;
990
991 skb = tcp_stream_alloc_skb(sk, 0, sk->sk_allocation,
992 tcp_rtx_and_write_queues_empty(sk));
993 if (!skb)
994 return NULL;
995
996#ifdef CONFIG_TLS_DEVICE
997 skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED);
998#endif
999 tcp_skb_entail(sk, skb);
1000 copy = size_goal;
1001 }
1002
1003 if (copy > *size)
1004 copy = *size;
1005
1006 i = skb_shinfo(skb)->nr_frags;
1007 can_coalesce = skb_can_coalesce(skb, i, page, offset);
1008 if (!can_coalesce && i >= READ_ONCE(sysctl_max_skb_frags)) {
1009 tcp_mark_push(tp, skb);
1010 goto new_segment;
1011 }
1012 if (tcp_downgrade_zcopy_pure(sk, skb))
1013 return NULL;
1014
1015 copy = tcp_wmem_schedule(sk, copy);
1016 if (!copy)
1017 return NULL;
1018
1019 if (can_coalesce) {
1020 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1021 } else {
1022 get_page(page);
1023 skb_fill_page_desc_noacc(skb, i, page, offset, copy);
1024 }
1025
1026 if (!(flags & MSG_NO_SHARED_FRAGS))
1027 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
1028
1029 skb->len += copy;
1030 skb->data_len += copy;
1031 skb->truesize += copy;
1032 sk_wmem_queued_add(sk, copy);
1033 sk_mem_charge(sk, copy);
1034 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1035 TCP_SKB_CB(skb)->end_seq += copy;
1036 tcp_skb_pcount_set(skb, 0);
1037
1038 *size = copy;
1039 return skb;
1040}
1041
1042ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
1043 size_t size, int flags)
1044{
1045 struct tcp_sock *tp = tcp_sk(sk);
1046 int mss_now, size_goal;
1047 int err;
1048 ssize_t copied;
1049 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1050
1051 if (IS_ENABLED(CONFIG_DEBUG_VM) &&
1052 WARN_ONCE(!sendpage_ok(page),
1053 "page must not be a Slab one and have page_count > 0"))
1054 return -EINVAL;
1055
1056 /* Wait for a connection to finish. One exception is TCP Fast Open
1057 * (passive side) where data is allowed to be sent before a connection
1058 * is fully established.
1059 */
1060 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1061 !tcp_passive_fastopen(sk)) {
1062 err = sk_stream_wait_connect(sk, &timeo);
1063 if (err != 0)
1064 goto out_err;
1065 }
1066
1067 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1068
1069 mss_now = tcp_send_mss(sk, &size_goal, flags);
1070 copied = 0;
1071
1072 err = -EPIPE;
1073 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1074 goto out_err;
1075
1076 while (size > 0) {
1077 struct sk_buff *skb;
1078 size_t copy = size;
1079
1080 skb = tcp_build_frag(sk, size_goal, flags, page, offset, &copy);
1081 if (!skb)
1082 goto wait_for_space;
1083
1084 if (!copied)
1085 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1086
1087 copied += copy;
1088 offset += copy;
1089 size -= copy;
1090 if (!size)
1091 goto out;
1092
1093 if (skb->len < size_goal || (flags & MSG_OOB))
1094 continue;
1095
1096 if (forced_push(tp)) {
1097 tcp_mark_push(tp, skb);
1098 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1099 } else if (skb == tcp_send_head(sk))
1100 tcp_push_one(sk, mss_now);
1101 continue;
1102
1103wait_for_space:
1104 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1105 tcp_push(sk, flags & ~MSG_MORE, mss_now,
1106 TCP_NAGLE_PUSH, size_goal);
1107
1108 err = sk_stream_wait_memory(sk, &timeo);
1109 if (err != 0)
1110 goto do_error;
1111
1112 mss_now = tcp_send_mss(sk, &size_goal, flags);
1113 }
1114
1115out:
1116 if (copied) {
1117 tcp_tx_timestamp(sk, sk->sk_tsflags);
1118 if (!(flags & MSG_SENDPAGE_NOTLAST))
1119 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1120 }
1121 return copied;
1122
1123do_error:
1124 tcp_remove_empty_skb(sk);
1125 if (copied)
1126 goto out;
1127out_err:
1128 /* make sure we wake any epoll edge trigger waiter */
1129 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1130 sk->sk_write_space(sk);
1131 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1132 }
1133 return sk_stream_error(sk, flags, err);
1134}
1135EXPORT_SYMBOL_GPL(do_tcp_sendpages);
1136
1137int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
1138 size_t size, int flags)
1139{
978int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
979 size_t size, int flags)
980{
981 struct bio_vec bvec;
982 struct msghdr msg = { .msg_flags = flags | MSG_SPLICE_PAGES, };
983
1140 if (!(sk->sk_route_caps & NETIF_F_SG))
1141 return sock_no_sendpage_locked(sk, page, offset, size, flags);
1142
1143 tcp_rate_check_app_limited(sk); /* is sending application-limited? */
1144
984 if (!(sk->sk_route_caps & NETIF_F_SG))
985 return sock_no_sendpage_locked(sk, page, offset, size, flags);
986
987 tcp_rate_check_app_limited(sk); /* is sending application-limited? */
988
1145 return do_tcp_sendpages(sk, page, offset, size, flags);
989 bvec_set_page(&bvec, page, size, offset);
990 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size);
991
992 if (flags & MSG_SENDPAGE_NOTLAST)
993 msg.msg_flags |= MSG_MORE;
994
995 return tcp_sendmsg_locked(sk, &msg, size);
1146}
1147EXPORT_SYMBOL_GPL(tcp_sendpage_locked);
1148
1149int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1150 size_t size, int flags)
1151{
1152 int ret;
1153

--- 64 unchanged lines hidden (view full) ---

1218{
1219 struct tcp_sock *tp = tcp_sk(sk);
1220 struct ubuf_info *uarg = NULL;
1221 struct sk_buff *skb;
1222 struct sockcm_cookie sockc;
1223 int flags, err, copied = 0;
1224 int mss_now = 0, size_goal, copied_syn = 0;
1225 int process_backlog = 0;
996}
997EXPORT_SYMBOL_GPL(tcp_sendpage_locked);
998
999int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1000 size_t size, int flags)
1001{
1002 int ret;
1003

--- 64 unchanged lines hidden (view full) ---

1068{
1069 struct tcp_sock *tp = tcp_sk(sk);
1070 struct ubuf_info *uarg = NULL;
1071 struct sk_buff *skb;
1072 struct sockcm_cookie sockc;
1073 int flags, err, copied = 0;
1074 int mss_now = 0, size_goal, copied_syn = 0;
1075 int process_backlog = 0;
1226 bool zc = false;
1076 int zc = 0;
1227 long timeo;
1228
1229 flags = msg->msg_flags;
1230
1231 if ((flags & MSG_ZEROCOPY) && size) {
1077 long timeo;
1078
1079 flags = msg->msg_flags;
1080
1081 if ((flags & MSG_ZEROCOPY) && size) {
1232 skb = tcp_write_queue_tail(sk);
1233
1234 if (msg->msg_ubuf) {
1235 uarg = msg->msg_ubuf;
1082 if (msg->msg_ubuf) {
1083 uarg = msg->msg_ubuf;
1236 net_zcopy_get(uarg);
1237 zc = sk->sk_route_caps & NETIF_F_SG;
1084 if (sk->sk_route_caps & NETIF_F_SG)
1085 zc = MSG_ZEROCOPY;
1238 } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
1086 } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
1087 skb = tcp_write_queue_tail(sk);
1239 uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb));
1240 if (!uarg) {
1241 err = -ENOBUFS;
1242 goto out_err;
1243 }
1088 uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb));
1089 if (!uarg) {
1090 err = -ENOBUFS;
1091 goto out_err;
1092 }
1244 zc = sk->sk_route_caps & NETIF_F_SG;
1245 if (!zc)
1093 if (sk->sk_route_caps & NETIF_F_SG)
1094 zc = MSG_ZEROCOPY;
1095 else
1246 uarg_to_msgzc(uarg)->zerocopy = 0;
1247 }
1096 uarg_to_msgzc(uarg)->zerocopy = 0;
1097 }
1098 } else if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES) && size) {
1099 if (sk->sk_route_caps & NETIF_F_SG)
1100 zc = MSG_SPLICE_PAGES;
1248 }
1249
1250 if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) &&
1251 !tp->repair) {
1252 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
1253 if (err == -EINPROGRESS && copied_syn > 0)
1254 goto out;
1255 else if (err)

--- 46 unchanged lines hidden (view full) ---

1302restart:
1303 mss_now = tcp_send_mss(sk, &size_goal, flags);
1304
1305 err = -EPIPE;
1306 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1307 goto do_error;
1308
1309 while (msg_data_left(msg)) {
1101 }
1102
1103 if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) &&
1104 !tp->repair) {
1105 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
1106 if (err == -EINPROGRESS && copied_syn > 0)
1107 goto out;
1108 else if (err)

--- 46 unchanged lines hidden (view full) ---

1155restart:
1156 mss_now = tcp_send_mss(sk, &size_goal, flags);
1157
1158 err = -EPIPE;
1159 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1160 goto do_error;
1161
1162 while (msg_data_left(msg)) {
1310 int copy = 0;
1163 ssize_t copy = 0;
1311
1312 skb = tcp_write_queue_tail(sk);
1313 if (skb)
1314 copy = size_goal - skb->len;
1315
1316 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1317 bool first_skb;
1318
1319new_segment:
1320 if (!sk_stream_memory_free(sk))
1321 goto wait_for_space;
1322
1323 if (unlikely(process_backlog >= 16)) {
1324 process_backlog = 0;
1325 if (sk_flush_backlog(sk))
1326 goto restart;
1327 }
1328 first_skb = tcp_rtx_and_write_queues_empty(sk);
1164
1165 skb = tcp_write_queue_tail(sk);
1166 if (skb)
1167 copy = size_goal - skb->len;
1168
1169 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1170 bool first_skb;
1171
1172new_segment:
1173 if (!sk_stream_memory_free(sk))
1174 goto wait_for_space;
1175
1176 if (unlikely(process_backlog >= 16)) {
1177 process_backlog = 0;
1178 if (sk_flush_backlog(sk))
1179 goto restart;
1180 }
1181 first_skb = tcp_rtx_and_write_queues_empty(sk);
1329 skb = tcp_stream_alloc_skb(sk, 0, sk->sk_allocation,
1182 skb = tcp_stream_alloc_skb(sk, sk->sk_allocation,
1330 first_skb);
1331 if (!skb)
1332 goto wait_for_space;
1333
1334 process_backlog++;
1335
1336 tcp_skb_entail(sk, skb);
1337 copy = size_goal;

--- 5 unchanged lines hidden (view full) ---

1343 if (tp->repair)
1344 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1345 }
1346
1347 /* Try to append data to the end of skb. */
1348 if (copy > msg_data_left(msg))
1349 copy = msg_data_left(msg);
1350
1183 first_skb);
1184 if (!skb)
1185 goto wait_for_space;
1186
1187 process_backlog++;
1188
1189 tcp_skb_entail(sk, skb);
1190 copy = size_goal;

--- 5 unchanged lines hidden (view full) ---

1196 if (tp->repair)
1197 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1198 }
1199
1200 /* Try to append data to the end of skb. */
1201 if (copy > msg_data_left(msg))
1202 copy = msg_data_left(msg);
1203
1351 if (!zc) {
1204 if (zc == 0) {
1352 bool merge = true;
1353 int i = skb_shinfo(skb)->nr_frags;
1354 struct page_frag *pfrag = sk_page_frag(sk);
1355
1356 if (!sk_page_frag_refill(sk, pfrag))
1357 goto wait_for_space;
1358
1359 if (!skb_can_coalesce(skb, i, pfrag->page,

--- 28 unchanged lines hidden (view full) ---

1388 if (merge) {
1389 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1390 } else {
1391 skb_fill_page_desc(skb, i, pfrag->page,
1392 pfrag->offset, copy);
1393 page_ref_inc(pfrag->page);
1394 }
1395 pfrag->offset += copy;
1205 bool merge = true;
1206 int i = skb_shinfo(skb)->nr_frags;
1207 struct page_frag *pfrag = sk_page_frag(sk);
1208
1209 if (!sk_page_frag_refill(sk, pfrag))
1210 goto wait_for_space;
1211
1212 if (!skb_can_coalesce(skb, i, pfrag->page,

--- 28 unchanged lines hidden (view full) ---

1241 if (merge) {
1242 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1243 } else {
1244 skb_fill_page_desc(skb, i, pfrag->page,
1245 pfrag->offset, copy);
1246 page_ref_inc(pfrag->page);
1247 }
1248 pfrag->offset += copy;
1396 } else {
1249 } else if (zc == MSG_ZEROCOPY) {
1397 /* First append to a fragless skb builds initial
1398 * pure zerocopy skb
1399 */
1400 if (!skb->len)
1401 skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY;
1402
1403 if (!skb_zcopy_pure(skb)) {
1404 copy = tcp_wmem_schedule(sk, copy);

--- 4 unchanged lines hidden (view full) ---

1409 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg);
1410 if (err == -EMSGSIZE || err == -EEXIST) {
1411 tcp_mark_push(tp, skb);
1412 goto new_segment;
1413 }
1414 if (err < 0)
1415 goto do_error;
1416 copy = err;
1250 /* First append to a fragless skb builds initial
1251 * pure zerocopy skb
1252 */
1253 if (!skb->len)
1254 skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY;
1255
1256 if (!skb_zcopy_pure(skb)) {
1257 copy = tcp_wmem_schedule(sk, copy);

--- 4 unchanged lines hidden (view full) ---

1262 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg);
1263 if (err == -EMSGSIZE || err == -EEXIST) {
1264 tcp_mark_push(tp, skb);
1265 goto new_segment;
1266 }
1267 if (err < 0)
1268 goto do_error;
1269 copy = err;
1270 } else if (zc == MSG_SPLICE_PAGES) {
1271 /* Splice in data if we can; copy if we can't. */
1272 if (tcp_downgrade_zcopy_pure(sk, skb))
1273 goto wait_for_space;
1274 copy = tcp_wmem_schedule(sk, copy);
1275 if (!copy)
1276 goto wait_for_space;
1277
1278 err = skb_splice_from_iter(skb, &msg->msg_iter, copy,
1279 sk->sk_allocation);
1280 if (err < 0) {
1281 if (err == -EMSGSIZE) {
1282 tcp_mark_push(tp, skb);
1283 goto new_segment;
1284 }
1285 goto do_error;
1286 }
1287 copy = err;
1288
1289 if (!(flags & MSG_NO_SHARED_FRAGS))
1290 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
1291
1292 sk_wmem_queued_add(sk, copy);
1293 sk_mem_charge(sk, copy);
1417 }
1418
1419 if (!copied)
1420 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1421
1422 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1423 TCP_SKB_CB(skb)->end_seq += copy;
1424 tcp_skb_pcount_set(skb, 0);

--- 29 unchanged lines hidden (view full) ---

1454 }
1455
1456out:
1457 if (copied) {
1458 tcp_tx_timestamp(sk, sockc.tsflags);
1459 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1460 }
1461out_nopush:
1294 }
1295
1296 if (!copied)
1297 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1298
1299 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1300 TCP_SKB_CB(skb)->end_seq += copy;
1301 tcp_skb_pcount_set(skb, 0);

--- 29 unchanged lines hidden (view full) ---

1331 }
1332
1333out:
1334 if (copied) {
1335 tcp_tx_timestamp(sk, sockc.tsflags);
1336 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1337 }
1338out_nopush:
1462 net_zcopy_put(uarg);
1339 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1340 if (uarg && !msg->msg_ubuf)
1341 net_zcopy_put(uarg);
1463 return copied + copied_syn;
1464
1465do_error:
1466 tcp_remove_empty_skb(sk);
1467
1468 if (copied + copied_syn)
1469 goto out;
1470out_err:
1342 return copied + copied_syn;
1343
1344do_error:
1345 tcp_remove_empty_skb(sk);
1346
1347 if (copied + copied_syn)
1348 goto out;
1349out_err:
1471 net_zcopy_put_abort(uarg, true);
1350 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1351 if (uarg && !msg->msg_ubuf)
1352 net_zcopy_put_abort(uarg, true);
1472 err = sk_stream_error(sk, flags, err);
1473 /* make sure we wake any epoll edge trigger waiter */
1474 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1475 sk->sk_write_space(sk);
1476 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1477 }
1478 return err;
1479}

--- 6 unchanged lines hidden (view full) ---

1486 lock_sock(sk);
1487 ret = tcp_sendmsg_locked(sk, msg, size);
1488 release_sock(sk);
1489
1490 return ret;
1491}
1492EXPORT_SYMBOL(tcp_sendmsg);
1493
1353 err = sk_stream_error(sk, flags, err);
1354 /* make sure we wake any epoll edge trigger waiter */
1355 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1356 sk->sk_write_space(sk);
1357 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1358 }
1359 return err;
1360}

--- 6 unchanged lines hidden (view full) ---

1367 lock_sock(sk);
1368 ret = tcp_sendmsg_locked(sk, msg, size);
1369 release_sock(sk);
1370
1371 return ret;
1372}
1373EXPORT_SYMBOL(tcp_sendmsg);
1374
1375void tcp_splice_eof(struct socket *sock)
1376{
1377 struct sock *sk = sock->sk;
1378 struct tcp_sock *tp = tcp_sk(sk);
1379 int mss_now, size_goal;
1380
1381 if (!tcp_write_queue_tail(sk))
1382 return;
1383
1384 lock_sock(sk);
1385 mss_now = tcp_send_mss(sk, &size_goal, 0);
1386 tcp_push(sk, 0, mss_now, tp->nonagle, size_goal);
1387 release_sock(sk);
1388}
1389EXPORT_SYMBOL_GPL(tcp_splice_eof);
1390
1494/*
1495 * Handle reading urgent data. BSD has very simple semantics for
1496 * this, no blocking and very strange errors 8)
1497 */
1498
1499static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1500{
1501 struct tcp_sock *tp = tcp_sk(sk);

--- 64 unchanged lines hidden (view full) ---

1566}
1567
1568/* Clean up the receive buffer for full frames taken by the user,
1569 * then send an ACK if necessary. COPIED is the number of bytes
1570 * tcp_recvmsg has given to the user so far, it speeds up the
1571 * calculation of whether or not we must ACK for the sake of
1572 * a window update.
1573 */
1391/*
1392 * Handle reading urgent data. BSD has very simple semantics for
1393 * this, no blocking and very strange errors 8)
1394 */
1395
1396static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1397{
1398 struct tcp_sock *tp = tcp_sk(sk);

--- 64 unchanged lines hidden (view full) ---

1463}
1464
1465/* Clean up the receive buffer for full frames taken by the user,
1466 * then send an ACK if necessary. COPIED is the number of bytes
1467 * tcp_recvmsg has given to the user so far, it speeds up the
1468 * calculation of whether or not we must ACK for the sake of
1469 * a window update.
1470 */
1574static void __tcp_cleanup_rbuf(struct sock *sk, int copied)
1471void __tcp_cleanup_rbuf(struct sock *sk, int copied)
1575{
1576 struct tcp_sock *tp = tcp_sk(sk);
1577 bool time_to_ack = false;
1578
1579 if (inet_csk_ack_scheduled(sk)) {
1580 const struct inet_connection_sock *icsk = inet_csk(sk);
1581
1582 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */

--- 185 unchanged lines hidden (view full) ---

1768 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1769 u8 tcp_flags;
1770 int used;
1771
1772 __skb_unlink(skb, &sk->sk_receive_queue);
1773 WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
1774 tcp_flags = TCP_SKB_CB(skb)->tcp_flags;
1775 used = recv_actor(sk, skb);
1472{
1473 struct tcp_sock *tp = tcp_sk(sk);
1474 bool time_to_ack = false;
1475
1476 if (inet_csk_ack_scheduled(sk)) {
1477 const struct inet_connection_sock *icsk = inet_csk(sk);
1478
1479 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */

--- 185 unchanged lines hidden (view full) ---

1665 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1666 u8 tcp_flags;
1667 int used;
1668
1669 __skb_unlink(skb, &sk->sk_receive_queue);
1670 WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
1671 tcp_flags = TCP_SKB_CB(skb)->tcp_flags;
1672 used = recv_actor(sk, skb);
1776 consume_skb(skb);
1777 if (used < 0) {
1778 if (!copied)
1779 copied = used;
1780 break;
1781 }
1782 seq += used;
1783 copied += used;
1784
1785 if (tcp_flags & TCPHDR_FIN) {
1786 ++seq;
1787 break;
1788 }
1789 }
1673 if (used < 0) {
1674 if (!copied)
1675 copied = used;
1676 break;
1677 }
1678 seq += used;
1679 copied += used;
1680
1681 if (tcp_flags & TCPHDR_FIN) {
1682 ++seq;
1683 break;
1684 }
1685 }
1790 WRITE_ONCE(tp->copied_seq, seq);
1791
1792 tcp_rcv_space_adjust(sk);
1793
1794 /* Clean up data we have read: This will do ACK frames. */
1795 if (copied > 0)
1796 __tcp_cleanup_rbuf(sk, copied);
1797
1798 return copied;
1799}
1800EXPORT_SYMBOL(tcp_read_skb);
1801
1802void tcp_read_done(struct sock *sk, size_t len)
1803{
1804 struct tcp_sock *tp = tcp_sk(sk);
1805 u32 seq = tp->copied_seq;

--- 1279 unchanged lines hidden (view full) ---

3085int tcp_disconnect(struct sock *sk, int flags)
3086{
3087 struct inet_sock *inet = inet_sk(sk);
3088 struct inet_connection_sock *icsk = inet_csk(sk);
3089 struct tcp_sock *tp = tcp_sk(sk);
3090 int old_state = sk->sk_state;
3091 u32 seq;
3092
1686 return copied;
1687}
1688EXPORT_SYMBOL(tcp_read_skb);
1689
1690void tcp_read_done(struct sock *sk, size_t len)
1691{
1692 struct tcp_sock *tp = tcp_sk(sk);
1693 u32 seq = tp->copied_seq;

--- 1279 unchanged lines hidden (view full) ---

2973int tcp_disconnect(struct sock *sk, int flags)
2974{
2975 struct inet_sock *inet = inet_sk(sk);
2976 struct inet_connection_sock *icsk = inet_csk(sk);
2977 struct tcp_sock *tp = tcp_sk(sk);
2978 int old_state = sk->sk_state;
2979 u32 seq;
2980
2981 /* Deny disconnect if other threads are blocked in sk_wait_event()
2982 * or inet_wait_for_connect().
2983 */
2984 if (sk->sk_wait_pending)
2985 return -EBUSY;
2986
3093 if (old_state != TCP_CLOSE)
3094 tcp_set_state(sk, TCP_CLOSE);
3095
3096 /* ABORT function of RFC793 */
3097 if (old_state == TCP_LISTEN) {
3098 inet_csk_listen_stop(sk);
3099 } else if (unlikely(tp->repair)) {
3100 WRITE_ONCE(sk->sk_err, ECONNABORTED);

--- 975 unchanged lines hidden (view full) ---

4076 len = min_t(unsigned int, len, sizeof(int));
4077
4078 if (len < 0)
4079 return -EINVAL;
4080
4081 switch (optname) {
4082 case TCP_MAXSEG:
4083 val = tp->mss_cache;
2987 if (old_state != TCP_CLOSE)
2988 tcp_set_state(sk, TCP_CLOSE);
2989
2990 /* ABORT function of RFC793 */
2991 if (old_state == TCP_LISTEN) {
2992 inet_csk_listen_stop(sk);
2993 } else if (unlikely(tp->repair)) {
2994 WRITE_ONCE(sk->sk_err, ECONNABORTED);

--- 975 unchanged lines hidden (view full) ---

3970 len = min_t(unsigned int, len, sizeof(int));
3971
3972 if (len < 0)
3973 return -EINVAL;
3974
3975 switch (optname) {
3976 case TCP_MAXSEG:
3977 val = tp->mss_cache;
4084 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3978 if (tp->rx_opt.user_mss &&
3979 ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
4085 val = tp->rx_opt.user_mss;
4086 if (tp->repair)
4087 val = tp->rx_opt.mss_clamp;
4088 break;
4089 case TCP_NODELAY:
4090 val = !!(tp->nonagle&TCP_NAGLE_OFF);
4091 break;
4092 case TCP_CORK:

--- 584 unchanged lines hidden (view full) ---

4677
4678 refcount_inc(&tw->tw_refcnt);
4679 local_bh_disable();
4680 inet_twsk_deschedule_put(tw);
4681 local_bh_enable();
4682 return 0;
4683 }
4684
3980 val = tp->rx_opt.user_mss;
3981 if (tp->repair)
3982 val = tp->rx_opt.mss_clamp;
3983 break;
3984 case TCP_NODELAY:
3985 val = !!(tp->nonagle&TCP_NAGLE_OFF);
3986 break;
3987 case TCP_CORK:

--- 584 unchanged lines hidden (view full) ---

4572
4573 refcount_inc(&tw->tw_refcnt);
4574 local_bh_disable();
4575 inet_twsk_deschedule_put(tw);
4576 local_bh_enable();
4577 return 0;
4578 }
4579
4685 /* Don't race with userspace socket closes such as tcp_close. */
4686 lock_sock(sk);
4580 /* BPF context ensures sock locking. */
4581 if (!has_current_bpf_ctx())
4582 /* Don't race with userspace socket closes such as tcp_close. */
4583 lock_sock(sk);
4687
4688 if (sk->sk_state == TCP_LISTEN) {
4689 tcp_set_state(sk, TCP_CLOSE);
4690 inet_csk_listen_stop(sk);
4691 }
4692
4693 /* Don't race with BH socket closes such as inet_csk_listen_stop. */
4694 local_bh_disable();

--- 7 unchanged lines hidden (view full) ---

4702 if (tcp_need_reset(sk->sk_state))
4703 tcp_send_active_reset(sk, GFP_ATOMIC);
4704 tcp_done(sk);
4705 }
4706
4707 bh_unlock_sock(sk);
4708 local_bh_enable();
4709 tcp_write_queue_purge(sk);
4584
4585 if (sk->sk_state == TCP_LISTEN) {
4586 tcp_set_state(sk, TCP_CLOSE);
4587 inet_csk_listen_stop(sk);
4588 }
4589
4590 /* Don't race with BH socket closes such as inet_csk_listen_stop. */
4591 local_bh_disable();

--- 7 unchanged lines hidden (view full) ---

4599 if (tcp_need_reset(sk->sk_state))
4600 tcp_send_active_reset(sk, GFP_ATOMIC);
4601 tcp_done(sk);
4602 }
4603
4604 bh_unlock_sock(sk);
4605 local_bh_enable();
4606 tcp_write_queue_purge(sk);
4710 release_sock(sk);
4607 if (!has_current_bpf_ctx())
4608 release_sock(sk);
4711 return 0;
4712}
4713EXPORT_SYMBOL_GPL(tcp_abort);
4714
4715extern struct tcp_congestion_ops tcp_reno;
4716
4717static __initdata unsigned long thash_entries;
4718static int __init set_thash_entries(char *str)

--- 122 unchanged lines hidden --- 		4609 return 0;
4610}
4611EXPORT_SYMBOL_GPL(tcp_abort);
4612
4613extern struct tcp_congestion_ops tcp_reno;
4614
4615static __initdata unsigned long thash_entries;
4616static int __init set_thash_entries(char *str)

--- 122 unchanged lines hidden ---