109903869SMartin KaFai Lau // SPDX-License-Identifier: GPL-2.0
209903869SMartin KaFai Lau /* Copyright (c) 2019 Facebook */
309903869SMartin KaFai Lau
409903869SMartin KaFai Lau /* WARNING: This implemenation is not necessarily the same
509903869SMartin KaFai Lau * as the tcp_dctcp.c. The purpose is mainly for testing
609903869SMartin KaFai Lau * the kernel BPF logic.
709903869SMartin KaFai Lau */
809903869SMartin KaFai Lau
9c9b24312SMartin KaFai Lau #include <stddef.h>
1009903869SMartin KaFai Lau #include <linux/bpf.h>
1109903869SMartin KaFai Lau #include <linux/types.h>
120c402c6cSMartin KaFai Lau #include <linux/stddef.h>
130c402c6cSMartin KaFai Lau #include <linux/tcp.h>
14*3411c5b6SMartin KaFai Lau #include <errno.h>
153e689141SToke Høiland-Jørgensen #include <bpf/bpf_helpers.h>
16df8ff353SAndrii Nakryiko #include <bpf/bpf_tracing.h>
1709903869SMartin KaFai Lau #include "bpf_tcp_helpers.h"
1809903869SMartin KaFai Lau
1909903869SMartin KaFai Lau char _license[] SEC("license") = "GPL";
2009903869SMartin KaFai Lau
21574ee209SMartin KaFai Lau volatile const char fallback[TCP_CA_NAME_MAX];
22574ee209SMartin KaFai Lau const char bpf_dctcp[] = "bpf_dctcp";
23574ee209SMartin KaFai Lau const char tcp_cdg[] = "cdg";
24574ee209SMartin KaFai Lau char cc_res[TCP_CA_NAME_MAX];
25574ee209SMartin KaFai Lau int tcp_cdg_res = 0;
26c9b24312SMartin KaFai Lau int stg_result = 0;
27*3411c5b6SMartin KaFai Lau int ebusy_cnt = 0;
28c9b24312SMartin KaFai Lau
29c9b24312SMartin KaFai Lau struct {
30c9b24312SMartin KaFai Lau __uint(type, BPF_MAP_TYPE_SK_STORAGE);
31c9b24312SMartin KaFai Lau __uint(map_flags, BPF_F_NO_PREALLOC);
32c9b24312SMartin KaFai Lau __type(key, int);
33c9b24312SMartin KaFai Lau __type(value, int);
34c9b24312SMartin KaFai Lau } sk_stg_map SEC(".maps");
35c9b24312SMartin KaFai Lau
3609903869SMartin KaFai Lau #define DCTCP_MAX_ALPHA 1024U
3709903869SMartin KaFai Lau
3809903869SMartin KaFai Lau struct dctcp {
3909903869SMartin KaFai Lau __u32 old_delivered;
4009903869SMartin KaFai Lau __u32 old_delivered_ce;
4109903869SMartin KaFai Lau __u32 prior_rcv_nxt;
4209903869SMartin KaFai Lau __u32 dctcp_alpha;
4309903869SMartin KaFai Lau __u32 next_seq;
4409903869SMartin KaFai Lau __u32 ce_state;
4509903869SMartin KaFai Lau __u32 loss_cwnd;
4609903869SMartin KaFai Lau };
4709903869SMartin KaFai Lau
4809903869SMartin KaFai Lau static unsigned int dctcp_shift_g = 4; /* g = 1/2^4 */
4909903869SMartin KaFai Lau static unsigned int dctcp_alpha_on_init = DCTCP_MAX_ALPHA;
5009903869SMartin KaFai Lau
dctcp_reset(const struct tcp_sock * tp,struct dctcp * ca)5109903869SMartin KaFai Lau static __always_inline void dctcp_reset(const struct tcp_sock *tp,
5209903869SMartin KaFai Lau struct dctcp *ca)
5309903869SMartin KaFai Lau {
5409903869SMartin KaFai Lau ca->next_seq = tp->snd_nxt;
5509903869SMartin KaFai Lau
5609903869SMartin KaFai Lau ca->old_delivered = tp->delivered;
5709903869SMartin KaFai Lau ca->old_delivered_ce = tp->delivered_ce;
5809903869SMartin KaFai Lau }
5909903869SMartin KaFai Lau
60ac065870SAndrii Nakryiko SEC("struct_ops/dctcp_init")
BPF_PROG(dctcp_init,struct sock * sk)61ac065870SAndrii Nakryiko void BPF_PROG(dctcp_init, struct sock *sk)
6209903869SMartin KaFai Lau {
6309903869SMartin KaFai Lau const struct tcp_sock *tp = tcp_sk(sk);
6409903869SMartin KaFai Lau struct dctcp *ca = inet_csk_ca(sk);
65c9b24312SMartin KaFai Lau int *stg;
6609903869SMartin KaFai Lau
67574ee209SMartin KaFai Lau if (!(tp->ecn_flags & TCP_ECN_OK) && fallback[0]) {
68574ee209SMartin KaFai Lau /* Switch to fallback */
69*3411c5b6SMartin KaFai Lau if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
70*3411c5b6SMartin KaFai Lau (void *)fallback, sizeof(fallback)) == -EBUSY)
71*3411c5b6SMartin KaFai Lau ebusy_cnt++;
72*3411c5b6SMartin KaFai Lau
73*3411c5b6SMartin KaFai Lau /* Switch back to myself and the recurred dctcp_init()
74*3411c5b6SMartin KaFai Lau * will get -EBUSY for all bpf_setsockopt(TCP_CONGESTION),
75*3411c5b6SMartin KaFai Lau * except the last "cdg" one.
76574ee209SMartin KaFai Lau */
77*3411c5b6SMartin KaFai Lau if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
78*3411c5b6SMartin KaFai Lau (void *)bpf_dctcp, sizeof(bpf_dctcp)) == -EBUSY)
79*3411c5b6SMartin KaFai Lau ebusy_cnt++;
80*3411c5b6SMartin KaFai Lau
81574ee209SMartin KaFai Lau /* Switch back to fallback */
82*3411c5b6SMartin KaFai Lau if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
83*3411c5b6SMartin KaFai Lau (void *)fallback, sizeof(fallback)) == -EBUSY)
84*3411c5b6SMartin KaFai Lau ebusy_cnt++;
85*3411c5b6SMartin KaFai Lau
86574ee209SMartin KaFai Lau /* Expecting -ENOTSUPP for tcp_cdg_res */
87574ee209SMartin KaFai Lau tcp_cdg_res = bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
88574ee209SMartin KaFai Lau (void *)tcp_cdg, sizeof(tcp_cdg));
89574ee209SMartin KaFai Lau bpf_getsockopt(sk, SOL_TCP, TCP_CONGESTION,
90574ee209SMartin KaFai Lau (void *)cc_res, sizeof(cc_res));
91574ee209SMartin KaFai Lau return;
92574ee209SMartin KaFai Lau }
93574ee209SMartin KaFai Lau
9409903869SMartin KaFai Lau ca->prior_rcv_nxt = tp->rcv_nxt;
9509903869SMartin KaFai Lau ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
9609903869SMartin KaFai Lau ca->loss_cwnd = 0;
9709903869SMartin KaFai Lau ca->ce_state = 0;
9809903869SMartin KaFai Lau
99c9b24312SMartin KaFai Lau stg = bpf_sk_storage_get(&sk_stg_map, (void *)tp, NULL, 0);
100c9b24312SMartin KaFai Lau if (stg) {
101c9b24312SMartin KaFai Lau stg_result = *stg;
102c9b24312SMartin KaFai Lau bpf_sk_storage_delete(&sk_stg_map, (void *)tp);
103c9b24312SMartin KaFai Lau }
10409903869SMartin KaFai Lau dctcp_reset(tp, ca);
10509903869SMartin KaFai Lau }
10609903869SMartin KaFai Lau
107ac065870SAndrii Nakryiko SEC("struct_ops/dctcp_ssthresh")
BPF_PROG(dctcp_ssthresh,struct sock * sk)108ac065870SAndrii Nakryiko __u32 BPF_PROG(dctcp_ssthresh, struct sock *sk)
10909903869SMartin KaFai Lau {
11009903869SMartin KaFai Lau struct dctcp *ca = inet_csk_ca(sk);
11109903869SMartin KaFai Lau struct tcp_sock *tp = tcp_sk(sk);
11209903869SMartin KaFai Lau
11309903869SMartin KaFai Lau ca->loss_cwnd = tp->snd_cwnd;
11409903869SMartin KaFai Lau return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
11509903869SMartin KaFai Lau }
11609903869SMartin KaFai Lau
117ac065870SAndrii Nakryiko SEC("struct_ops/dctcp_update_alpha")
BPF_PROG(dctcp_update_alpha,struct sock * sk,__u32 flags)118ac065870SAndrii Nakryiko void BPF_PROG(dctcp_update_alpha, struct sock *sk, __u32 flags)
11909903869SMartin KaFai Lau {
12009903869SMartin KaFai Lau const struct tcp_sock *tp = tcp_sk(sk);
12109903869SMartin KaFai Lau struct dctcp *ca = inet_csk_ca(sk);
12209903869SMartin KaFai Lau
12309903869SMartin KaFai Lau /* Expired RTT */
12409903869SMartin KaFai Lau if (!before(tp->snd_una, ca->next_seq)) {
12509903869SMartin KaFai Lau __u32 delivered_ce = tp->delivered_ce - ca->old_delivered_ce;
12609903869SMartin KaFai Lau __u32 alpha = ca->dctcp_alpha;
12709903869SMartin KaFai Lau
12809903869SMartin KaFai Lau /* alpha = (1 - g) * alpha + g * F */
12909903869SMartin KaFai Lau
13009903869SMartin KaFai Lau alpha -= min_not_zero(alpha, alpha >> dctcp_shift_g);
13109903869SMartin KaFai Lau if (delivered_ce) {
13209903869SMartin KaFai Lau __u32 delivered = tp->delivered - ca->old_delivered;
13309903869SMartin KaFai Lau
13409903869SMartin KaFai Lau /* If dctcp_shift_g == 1, a 32bit value would overflow
13509903869SMartin KaFai Lau * after 8 M packets.
13609903869SMartin KaFai Lau */
13709903869SMartin KaFai Lau delivered_ce <<= (10 - dctcp_shift_g);
13809903869SMartin KaFai Lau delivered_ce /= max(1U, delivered);
13909903869SMartin KaFai Lau
14009903869SMartin KaFai Lau alpha = min(alpha + delivered_ce, DCTCP_MAX_ALPHA);
14109903869SMartin KaFai Lau }
14209903869SMartin KaFai Lau ca->dctcp_alpha = alpha;
14309903869SMartin KaFai Lau dctcp_reset(tp, ca);
14409903869SMartin KaFai Lau }
14509903869SMartin KaFai Lau }
14609903869SMartin KaFai Lau
dctcp_react_to_loss(struct sock * sk)14709903869SMartin KaFai Lau static __always_inline void dctcp_react_to_loss(struct sock *sk)
14809903869SMartin KaFai Lau {
14909903869SMartin KaFai Lau struct dctcp *ca = inet_csk_ca(sk);
15009903869SMartin KaFai Lau struct tcp_sock *tp = tcp_sk(sk);
15109903869SMartin KaFai Lau
15209903869SMartin KaFai Lau ca->loss_cwnd = tp->snd_cwnd;
15309903869SMartin KaFai Lau tp->snd_ssthresh = max(tp->snd_cwnd >> 1U, 2U);
15409903869SMartin KaFai Lau }
15509903869SMartin KaFai Lau
156ac065870SAndrii Nakryiko SEC("struct_ops/dctcp_state")
BPF_PROG(dctcp_state,struct sock * sk,__u8 new_state)157ac065870SAndrii Nakryiko void BPF_PROG(dctcp_state, struct sock *sk, __u8 new_state)
15809903869SMartin KaFai Lau {
15909903869SMartin KaFai Lau if (new_state == TCP_CA_Recovery &&
16009903869SMartin KaFai Lau new_state != BPF_CORE_READ_BITFIELD(inet_csk(sk), icsk_ca_state))
16109903869SMartin KaFai Lau dctcp_react_to_loss(sk);
16209903869SMartin KaFai Lau /* We handle RTO in dctcp_cwnd_event to ensure that we perform only
16309903869SMartin KaFai Lau * one loss-adjustment per RTT.
16409903869SMartin KaFai Lau */
16509903869SMartin KaFai Lau }
16609903869SMartin KaFai Lau
dctcp_ece_ack_cwr(struct sock * sk,__u32 ce_state)16709903869SMartin KaFai Lau static __always_inline void dctcp_ece_ack_cwr(struct sock *sk, __u32 ce_state)
16809903869SMartin KaFai Lau {
16909903869SMartin KaFai Lau struct tcp_sock *tp = tcp_sk(sk);
17009903869SMartin KaFai Lau
17109903869SMartin KaFai Lau if (ce_state == 1)
17209903869SMartin KaFai Lau tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
17309903869SMartin KaFai Lau else
17409903869SMartin KaFai Lau tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
17509903869SMartin KaFai Lau }
17609903869SMartin KaFai Lau
17709903869SMartin KaFai Lau /* Minimal DCTP CE state machine:
17809903869SMartin KaFai Lau *
17909903869SMartin KaFai Lau * S: 0 <- last pkt was non-CE
18009903869SMartin KaFai Lau * 1 <- last pkt was CE
18109903869SMartin KaFai Lau */
18209903869SMartin KaFai Lau static __always_inline
dctcp_ece_ack_update(struct sock * sk,enum tcp_ca_event evt,__u32 * prior_rcv_nxt,__u32 * ce_state)18309903869SMartin KaFai Lau void dctcp_ece_ack_update(struct sock *sk, enum tcp_ca_event evt,
18409903869SMartin KaFai Lau __u32 *prior_rcv_nxt, __u32 *ce_state)
18509903869SMartin KaFai Lau {
18609903869SMartin KaFai Lau __u32 new_ce_state = (evt == CA_EVENT_ECN_IS_CE) ? 1 : 0;
18709903869SMartin KaFai Lau
18809903869SMartin KaFai Lau if (*ce_state != new_ce_state) {
18909903869SMartin KaFai Lau /* CE state has changed, force an immediate ACK to
19009903869SMartin KaFai Lau * reflect the new CE state. If an ACK was delayed,
19109903869SMartin KaFai Lau * send that first to reflect the prior CE state.
19209903869SMartin KaFai Lau */
19309903869SMartin KaFai Lau if (inet_csk(sk)->icsk_ack.pending & ICSK_ACK_TIMER) {
19409903869SMartin KaFai Lau dctcp_ece_ack_cwr(sk, *ce_state);
19509903869SMartin KaFai Lau bpf_tcp_send_ack(sk, *prior_rcv_nxt);
19609903869SMartin KaFai Lau }
19709903869SMartin KaFai Lau inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
19809903869SMartin KaFai Lau }
19909903869SMartin KaFai Lau *prior_rcv_nxt = tcp_sk(sk)->rcv_nxt;
20009903869SMartin KaFai Lau *ce_state = new_ce_state;
20109903869SMartin KaFai Lau dctcp_ece_ack_cwr(sk, new_ce_state);
20209903869SMartin KaFai Lau }
20309903869SMartin KaFai Lau
204ac065870SAndrii Nakryiko SEC("struct_ops/dctcp_cwnd_event")
BPF_PROG(dctcp_cwnd_event,struct sock * sk,enum tcp_ca_event ev)205ac065870SAndrii Nakryiko void BPF_PROG(dctcp_cwnd_event, struct sock *sk, enum tcp_ca_event ev)
20609903869SMartin KaFai Lau {
20709903869SMartin KaFai Lau struct dctcp *ca = inet_csk_ca(sk);
20809903869SMartin KaFai Lau
20909903869SMartin KaFai Lau switch (ev) {
21009903869SMartin KaFai Lau case CA_EVENT_ECN_IS_CE:
21109903869SMartin KaFai Lau case CA_EVENT_ECN_NO_CE:
21209903869SMartin KaFai Lau dctcp_ece_ack_update(sk, ev, &ca->prior_rcv_nxt, &ca->ce_state);
21309903869SMartin KaFai Lau break;
21409903869SMartin KaFai Lau case CA_EVENT_LOSS:
21509903869SMartin KaFai Lau dctcp_react_to_loss(sk);
21609903869SMartin KaFai Lau break;
21709903869SMartin KaFai Lau default:
21809903869SMartin KaFai Lau /* Don't care for the rest. */
21909903869SMartin KaFai Lau break;
22009903869SMartin KaFai Lau }
22109903869SMartin KaFai Lau }
22209903869SMartin KaFai Lau
223ac065870SAndrii Nakryiko SEC("struct_ops/dctcp_cwnd_undo")
BPF_PROG(dctcp_cwnd_undo,struct sock * sk)224ac065870SAndrii Nakryiko __u32 BPF_PROG(dctcp_cwnd_undo, struct sock *sk)
22509903869SMartin KaFai Lau {
22609903869SMartin KaFai Lau const struct dctcp *ca = inet_csk_ca(sk);
22709903869SMartin KaFai Lau
22809903869SMartin KaFai Lau return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
22909903869SMartin KaFai Lau }
23009903869SMartin KaFai Lau
23178e60bbbSMartin KaFai Lau extern void tcp_reno_cong_avoid(struct sock *sk, __u32 ack, __u32 acked) __ksym;
23278e60bbbSMartin KaFai Lau
23378e60bbbSMartin KaFai Lau SEC("struct_ops/dctcp_reno_cong_avoid")
BPF_PROG(dctcp_cong_avoid,struct sock * sk,__u32 ack,__u32 acked)23478e60bbbSMartin KaFai Lau void BPF_PROG(dctcp_cong_avoid, struct sock *sk, __u32 ack, __u32 acked)
23509903869SMartin KaFai Lau {
23678e60bbbSMartin KaFai Lau tcp_reno_cong_avoid(sk, ack, acked);
23709903869SMartin KaFai Lau }
23809903869SMartin KaFai Lau
23909903869SMartin KaFai Lau SEC(".struct_ops")
24009903869SMartin KaFai Lau struct tcp_congestion_ops dctcp_nouse = {
24109903869SMartin KaFai Lau .init = (void *)dctcp_init,
24209903869SMartin KaFai Lau .set_state = (void *)dctcp_state,
24309903869SMartin KaFai Lau .flags = TCP_CONG_NEEDS_ECN,
24409903869SMartin KaFai Lau .name = "bpf_dctcp_nouse",
24509903869SMartin KaFai Lau };
24609903869SMartin KaFai Lau
24709903869SMartin KaFai Lau SEC(".struct_ops")
24809903869SMartin KaFai Lau struct tcp_congestion_ops dctcp = {
24909903869SMartin KaFai Lau .init = (void *)dctcp_init,
25009903869SMartin KaFai Lau .in_ack_event = (void *)dctcp_update_alpha,
25109903869SMartin KaFai Lau .cwnd_event = (void *)dctcp_cwnd_event,
25209903869SMartin KaFai Lau .ssthresh = (void *)dctcp_ssthresh,
25378e60bbbSMartin KaFai Lau .cong_avoid = (void *)dctcp_cong_avoid,
25409903869SMartin KaFai Lau .undo_cwnd = (void *)dctcp_cwnd_undo,
25509903869SMartin KaFai Lau .set_state = (void *)dctcp_state,
25609903869SMartin KaFai Lau .flags = TCP_CONG_NEEDS_ECN,
25709903869SMartin KaFai Lau .name = "bpf_dctcp",
25809903869SMartin KaFai Lau };
259