tcp_output.c (5f6188a8003d080e3753b8f14f4a5a2325ae1ff6) tcp_output.c (76a9ebe811fb3d0605cb084f1ae6be5610541865)
1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Implementation of the Transmission Control Protocol(TCP).
7 *
8 * Authors: Ross Biro

--- 977 unchanged lines hidden (view full) ---

986}
987
988static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb)
989{
990 struct tcp_sock *tp = tcp_sk(sk);
991
992 skb->skb_mstamp_ns = tp->tcp_wstamp_ns;
993 if (sk->sk_pacing_status != SK_PACING_NONE) {
1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Implementation of the Transmission Control Protocol(TCP).
7 *
8 * Authors: Ross Biro

--- 977 unchanged lines hidden (view full) ---

986}
987
988static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb)
989{
990 struct tcp_sock *tp = tcp_sk(sk);
991
992 skb->skb_mstamp_ns = tp->tcp_wstamp_ns;
993 if (sk->sk_pacing_status != SK_PACING_NONE) {
994 u32 rate = sk->sk_pacing_rate;
994 unsigned long rate = sk->sk_pacing_rate;
995
996 /* Original sch_fq does not pace first 10 MSS
997 * Note that tp->data_segs_out overflows after 2^32 packets,
998 * this is a minor annoyance.
999 */
995
996 /* Original sch_fq does not pace first 10 MSS
997 * Note that tp->data_segs_out overflows after 2^32 packets,
998 * this is a minor annoyance.
999 */
1000 if (rate != ~0U && rate && tp->data_segs_out >= 10) {
1001 tp->tcp_wstamp_ns += div_u64((u64)skb->len * NSEC_PER_SEC, rate);
1000 if (rate != ~0UL && rate && tp->data_segs_out >= 10) {
1001 tp->tcp_wstamp_ns += div64_ul((u64)skb->len * NSEC_PER_SEC, rate);
1002
1003 tcp_internal_pacing(sk);
1004 }
1005 }
1006 list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
1007}
1008
1009/* This routine actually transmits TCP packets queued in by

--- 689 unchanged lines hidden (view full) ---

1699/* Return how many segs we'd like on a TSO packet,
1700 * to send one TSO packet per ms
1701 */
1702static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
1703 int min_tso_segs)
1704{
1705 u32 bytes, segs;
1706
1002
1003 tcp_internal_pacing(sk);
1004 }
1005 }
1006 list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
1007}
1008
1009/* This routine actually transmits TCP packets queued in by

--- 689 unchanged lines hidden (view full) ---

1699/* Return how many segs we'd like on a TSO packet,
1700 * to send one TSO packet per ms
1701 */
1702static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
1703 int min_tso_segs)
1704{
1705 u32 bytes, segs;
1706
1707 bytes = min(sk->sk_pacing_rate >> sk->sk_pacing_shift,
1708 sk->sk_gso_max_size - 1 - MAX_TCP_HEADER);
1707 bytes = min_t(unsigned long,
1708 sk->sk_pacing_rate >> sk->sk_pacing_shift,
1709 sk->sk_gso_max_size - 1 - MAX_TCP_HEADER);
1709
1710 /* Goal is to send at least one packet per ms,
1711 * not one big TSO packet every 100 ms.
1712 * This preserves ACK clocking and is consistent
1713 * with tcp_tso_should_defer() heuristic.
1714 */
1715 segs = max_t(u32, bytes / mss_now, min_tso_segs);
1716

--- 476 unchanged lines hidden (view full) ---

2193 * - high rates
2194 * Alas, some drivers / subsystems require a fair amount
2195 * of queued bytes to ensure line rate.
2196 * One example is wifi aggregation (802.11 AMPDU)
2197 */
2198static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
2199 unsigned int factor)
2200{
1710
1711 /* Goal is to send at least one packet per ms,
1712 * not one big TSO packet every 100 ms.
1713 * This preserves ACK clocking and is consistent
1714 * with tcp_tso_should_defer() heuristic.
1715 */
1716 segs = max_t(u32, bytes / mss_now, min_tso_segs);
1717

--- 476 unchanged lines hidden (view full) ---

2194 * - high rates
2195 * Alas, some drivers / subsystems require a fair amount
2196 * of queued bytes to ensure line rate.
2197 * One example is wifi aggregation (802.11 AMPDU)
2198 */
2199static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
2200 unsigned int factor)
2201{
2201 unsigned int limit;
2202 unsigned long limit;
2202
2203
2203 limit = max(2 * skb->truesize, sk->sk_pacing_rate >> sk->sk_pacing_shift);
2204 limit = min_t(u32, limit,
2204 limit = max_t(unsigned long,
2205 2 * skb->truesize,
2206 sk->sk_pacing_rate >> sk->sk_pacing_shift);
2207 limit = min_t(unsigned long, limit,
2205 sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes);
2206 limit <<= factor;
2207
2208 if (refcount_read(&sk->sk_wmem_alloc) > limit) {
2209 /* Always send skb if rtx queue is empty.
2210 * No need to wait for TX completion to call us back,
2211 * after softirq/tasklet schedule.
2212 * This helps when TX completions are delayed too much.

--- 1553 unchanged lines hidden --- 		2208 sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes);
2209 limit <<= factor;
2210
2211 if (refcount_read(&sk->sk_wmem_alloc) > limit) {
2212 /* Always send skb if rtx queue is empty.
2213 * No need to wait for TX completion to call us back,
2214 * after softirq/tasklet schedule.
2215 * This helps when TX completions are delayed too much.

--- 1553 unchanged lines hidden ---