Lines Matching +full:static +full:- +full:beta
1 // SPDX-License-Identifier: GPL-2.0-only
13 * "while (ca->ack_cnt > delta)" loop is changed to the equivalent
14 * "ca->ack_cnt / delta" operation.
26 #define BICTCP_BETA_SCALE 1024 /* Scale factor beta calculation
27 * max_cwnd = snd_cwnd * beta
41 static int fast_convergence = 1;
42 static const int beta = 717; /* = 717/1024 (BICTCP_BETA_SCALE) */ variable
43 static int initial_ssthresh;
44 static const int bic_scale = 41;
45 static int tcp_friendliness = 1;
47 static int hystart = 1;
48 static int hystart_detect = HYSTART_ACK_TRAIN | HYSTART_DELAY;
49 static int hystart_low_window = 16;
50 static int hystart_ack_delta_us = 2000;
52 static const __u32 cube_rtt_scale = (bic_scale * 10); /* 1024*c/rtt */
53 static const __u32 beta_scale = 8*(BICTCP_BETA_SCALE+beta) / 3
54 / (BICTCP_BETA_SCALE - beta);
55 /* calculate the "K" for (wmax-cwnd) = c/rtt * K^3
56 * so K = cubic_root( (wmax-cwnd)*rtt/c )
65 * HZ < 1,000,00 (corresponding to 10 nano-second)
69 static const __u64 cube_factor = (__u64)(1ull << (10+3*BICTCP_HZ))
94 static inline void bictcp_reset(struct bictcp *ca) in bictcp_reset()
96 ca->cnt = 0; in bictcp_reset()
97 ca->last_max_cwnd = 0; in bictcp_reset()
98 ca->last_cwnd = 0; in bictcp_reset()
99 ca->last_time = 0; in bictcp_reset()
100 ca->bic_origin_point = 0; in bictcp_reset()
101 ca->bic_K = 0; in bictcp_reset()
102 ca->delay_min = 0; in bictcp_reset()
103 ca->epoch_start = 0; in bictcp_reset()
104 ca->ack_cnt = 0; in bictcp_reset()
105 ca->tcp_cwnd = 0; in bictcp_reset()
106 ca->found = 0; in bictcp_reset()
115 static __always_inline __u64 div64_u64(__u64 dividend, __u64 divisor) in div64_u64()
123 static __always_inline int fls64(__u64 x) in fls64()
125 int num = BITS_PER_U64 - 1; in fls64()
130 if (!(x & (~0ull << (BITS_PER_U64-32)))) { in fls64()
131 num -= 32; in fls64()
134 if (!(x & (~0ull << (BITS_PER_U64-16)))) { in fls64()
135 num -= 16; in fls64()
138 if (!(x & (~0ull << (BITS_PER_U64-8)))) { in fls64()
139 num -= 8; in fls64()
142 if (!(x & (~0ull << (BITS_PER_U64-4)))) { in fls64()
143 num -= 4; in fls64()
146 if (!(x & (~0ull << (BITS_PER_U64-2)))) { in fls64()
147 num -= 2; in fls64()
150 if (!(x & (~0ull << (BITS_PER_U64-1)))) in fls64()
151 num -= 1; in fls64()
156 static __always_inline __u32 bictcp_clock_us(const struct sock *sk) in bictcp_clock_us()
158 return tcp_sk(sk)->tcp_mstamp; in bictcp_clock_us()
161 static __always_inline void bictcp_hystart_reset(struct sock *sk) in bictcp_hystart_reset()
166 ca->round_start = ca->last_ack = bictcp_clock_us(sk); in bictcp_hystart_reset()
167 ca->end_seq = tp->snd_nxt; in bictcp_hystart_reset()
168 ca->curr_rtt = ~0U; in bictcp_hystart_reset()
169 ca->sample_cnt = 0; in bictcp_hystart_reset()
184 tcp_sk(sk)->snd_ssthresh = initial_ssthresh; in BPF_PROG()
196 delta = now - tcp_sk(sk)->lsndtime; in BPF_PROG()
201 if (ca->epoch_start && delta > 0) { in BPF_PROG()
202 ca->epoch_start += delta; in BPF_PROG()
203 if (after(ca->epoch_start, now)) in BPF_PROG()
204 ca->epoch_start = now; in BPF_PROG()
212 * Precomputed then refined by hand - Willy Tarreau
215 * v = cbrt(x << 18) - 1
218 static const __u8 v[] = {
230 * Newton-Raphson iteration.
233 static __always_inline __u32 cubic_root(__u64 a) in cubic_root()
243 b = ((b * 84) >> 8) - 1; in cubic_root()
253 * Newton-Raphson iteration in cubic_root()
258 x = (2 * x + (__u32)div64_u64(a, (__u64)x * (__u64)(x - 1))); in cubic_root()
266 static __always_inline void bictcp_update(struct bictcp *ca, __u32 cwnd, in bictcp_update()
272 ca->ack_cnt += acked; /* count the number of ACKed packets */ in bictcp_update()
274 if (ca->last_cwnd == cwnd && in bictcp_update()
275 (__s32)(tcp_jiffies32 - ca->last_time) <= HZ / 32) in bictcp_update()
278 /* The CUBIC function can update ca->cnt at most once per jiffy. in bictcp_update()
279 * On all cwnd reduction events, ca->epoch_start is set to 0, in bictcp_update()
280 * which will force a recalculation of ca->cnt. in bictcp_update()
282 if (ca->epoch_start && tcp_jiffies32 == ca->last_time) in bictcp_update()
285 ca->last_cwnd = cwnd; in bictcp_update()
286 ca->last_time = tcp_jiffies32; in bictcp_update()
288 if (ca->epoch_start == 0) { in bictcp_update()
289 ca->epoch_start = tcp_jiffies32; /* record beginning */ in bictcp_update()
290 ca->ack_cnt = acked; /* start counting */ in bictcp_update()
291 ca->tcp_cwnd = cwnd; /* syn with cubic */ in bictcp_update()
293 if (ca->last_max_cwnd <= cwnd) { in bictcp_update()
294 ca->bic_K = 0; in bictcp_update()
295 ca->bic_origin_point = cwnd; in bictcp_update()
298 * (wmax-cwnd) * (srtt>>3 / HZ) / c * 2^(3*bictcp_HZ) in bictcp_update()
300 ca->bic_K = cubic_root(cube_factor in bictcp_update()
301 * (ca->last_max_cwnd - cwnd)); in bictcp_update()
302 ca->bic_origin_point = ca->last_max_cwnd; in bictcp_update()
306 /* cubic function - calc*/ in bictcp_update()
313 * time = (t - K) / 2^bictcp_HZ in bictcp_update()
320 t = (__s32)(tcp_jiffies32 - ca->epoch_start) * USEC_PER_JIFFY; in bictcp_update()
321 t += ca->delay_min; in bictcp_update()
326 if (t < ca->bic_K) /* t - K */ in bictcp_update()
327 offs = ca->bic_K - t; in bictcp_update()
329 offs = t - ca->bic_K; in bictcp_update()
331 /* c/rtt * (t-K)^3 */ in bictcp_update()
333 if (t < ca->bic_K) /* below origin*/ in bictcp_update()
334 bic_target = ca->bic_origin_point - delta; in bictcp_update()
336 bic_target = ca->bic_origin_point + delta; in bictcp_update()
338 /* cubic function - calc bictcp_cnt*/ in bictcp_update()
340 ca->cnt = cwnd / (bic_target - cwnd); in bictcp_update()
342 ca->cnt = 100 * cwnd; /* very small increment*/ in bictcp_update()
349 if (ca->last_max_cwnd == 0 && ca->cnt > 20) in bictcp_update()
350 ca->cnt = 20; /* increase cwnd 5% per RTT */ in bictcp_update()
360 if (ca->ack_cnt > delta && delta) { in bictcp_update()
361 n = ca->ack_cnt / delta; in bictcp_update()
362 ca->ack_cnt -= n * delta; in bictcp_update()
363 ca->tcp_cwnd += n; in bictcp_update()
366 if (ca->tcp_cwnd > cwnd) { /* if bic is slower than tcp */ in bictcp_update()
367 delta = ca->tcp_cwnd - cwnd; in bictcp_update()
369 if (ca->cnt > max_cnt) in bictcp_update()
370 ca->cnt = max_cnt; in bictcp_update()
377 ca->cnt = max(ca->cnt, 2U); in bictcp_update()
390 if (hystart && after(ack, ca->end_seq)) in BPF_STRUCT_OPS()
396 bictcp_update(ca, tp->snd_cwnd, acked); in BPF_STRUCT_OPS()
397 tcp_cong_avoid_ai(tp, ca->cnt, acked); in BPF_STRUCT_OPS()
405 ca->epoch_start = 0; /* end of epoch */ in BPF_STRUCT_OPS()
408 if (tp->snd_cwnd < ca->last_max_cwnd && fast_convergence) in BPF_STRUCT_OPS()
409 ca->last_max_cwnd = (tp->snd_cwnd * (BICTCP_BETA_SCALE + beta)) in BPF_STRUCT_OPS()
412 ca->last_max_cwnd = tp->snd_cwnd; in BPF_STRUCT_OPS()
414 return max((tp->snd_cwnd * beta) / BICTCP_BETA_SCALE, 2U); in BPF_STRUCT_OPS()
429 * slow start we begin with small TSO packets and ca->delay_min would
436 static __always_inline __u32 hystart_ack_delay(struct sock *sk) in hystart_ack_delay()
440 rate = sk->sk_pacing_rate; in hystart_ack_delay()
447 static __always_inline void hystart_update(struct sock *sk, __u32 delay) in hystart_update()
456 /* first detection parameter - ack-train detection */ in hystart_update()
457 if ((__s32)(now - ca->last_ack) <= hystart_ack_delta_us) { in hystart_update()
458 ca->last_ack = now; in hystart_update()
460 threshold = ca->delay_min + hystart_ack_delay(sk); in hystart_update()
463 * ca->delay_min/2. in hystart_update()
467 if (sk->sk_pacing_status == SK_PACING_NONE) in hystart_update()
470 if ((__s32)(now - ca->round_start) > threshold) { in hystart_update()
471 ca->found = 1; in hystart_update()
472 tp->snd_ssthresh = tp->snd_cwnd; in hystart_update()
479 if (ca->curr_rtt > delay) in hystart_update()
480 ca->curr_rtt = delay; in hystart_update()
481 if (ca->sample_cnt < HYSTART_MIN_SAMPLES) { in hystart_update()
482 ca->sample_cnt++; in hystart_update()
484 if (ca->curr_rtt > ca->delay_min + in hystart_update()
485 HYSTART_DELAY_THRESH(ca->delay_min >> 3)) { in hystart_update()
486 ca->found = 1; in hystart_update()
487 tp->snd_ssthresh = tp->snd_cwnd; in hystart_update()
504 if (sample->rtt_us < 0) in BPF_STRUCT_OPS()
508 if (ca->epoch_start && (__s32)(tcp_jiffies32 - ca->epoch_start) < HZ) in BPF_STRUCT_OPS()
511 delay = sample->rtt_us; in BPF_STRUCT_OPS()
516 if (ca->delay_min == 0 || ca->delay_min > delay) in BPF_STRUCT_OPS()
517 ca->delay_min = delay; in BPF_STRUCT_OPS()
520 if (!ca->found && tcp_in_slow_start(tp) && hystart && in BPF_STRUCT_OPS()
521 tp->snd_cwnd >= hystart_low_window) in BPF_STRUCT_OPS()