1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Kernel Connection Multiplexor 4 * 5 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com> 6 */ 7 8 #ifndef __NET_KCM_H_ 9 #define __NET_KCM_H_ 10 11 #include <linux/skbuff.h> 12 #include <net/sock.h> 13 #include <net/strparser.h> 14 #include <uapi/linux/kcm.h> 15 16 extern unsigned int kcm_net_id; 17 18 #define KCM_STATS_ADD(stat, count) ((stat) += (count)) 19 #define KCM_STATS_INCR(stat) ((stat)++) 20 21 struct kcm_psock_stats { 22 unsigned long long tx_msgs; 23 unsigned long long tx_bytes; 24 unsigned long long reserved; 25 unsigned long long unreserved; 26 unsigned int tx_aborts; 27 }; 28 29 struct kcm_mux_stats { 30 unsigned long long rx_msgs; 31 unsigned long long rx_bytes; 32 unsigned long long tx_msgs; 33 unsigned long long tx_bytes; 34 unsigned int rx_ready_drops; 35 unsigned int tx_retries; 36 unsigned int psock_attach; 37 unsigned int psock_unattach_rsvd; 38 unsigned int psock_unattach; 39 }; 40 41 struct kcm_stats { 42 unsigned long long rx_msgs; 43 unsigned long long rx_bytes; 44 unsigned long long tx_msgs; 45 unsigned long long tx_bytes; 46 }; 47 48 struct kcm_tx_msg { 49 unsigned int sent; 50 unsigned int frag_offset; 51 unsigned int msg_flags; 52 bool started_tx; 53 struct sk_buff *frag_skb; 54 struct sk_buff *last_skb; 55 }; 56 57 /* Socket structure for KCM client sockets */ 58 struct kcm_sock { 59 struct sock sk; 60 struct kcm_mux *mux; 61 struct list_head kcm_sock_list; 62 int index; 63 u32 done : 1; 64 struct work_struct done_work; 65 66 struct kcm_stats stats; 67 68 /* Transmit */ 69 struct kcm_psock *tx_psock; 70 struct work_struct tx_work; 71 struct list_head wait_psock_list; 72 struct sk_buff *seq_skb; 73 struct mutex tx_mutex; 74 u32 tx_stopped : 1; 75 76 /* Don't use bit fields here, these are set under different locks */ 77 bool tx_wait; 78 bool tx_wait_more; 79 80 /* Receive */ 81 struct kcm_psock *rx_psock; 82 struct list_head wait_rx_list; /* KCMs waiting for receiving */ 83 bool rx_wait; 84 u32 rx_disabled : 1; 85 }; 86 87 struct bpf_prog; 88 89 /* Structure for an attached lower socket */ 90 struct kcm_psock { 91 struct sock *sk; 92 struct strparser strp; 93 struct kcm_mux *mux; 94 int index; 95 96 u32 tx_stopped : 1; 97 u32 done : 1; 98 u32 unattaching : 1; 99 100 void (*save_state_change)(struct sock *sk); 101 void (*save_data_ready)(struct sock *sk); 102 void (*save_write_space)(struct sock *sk); 103 104 struct list_head psock_list; 105 106 struct kcm_psock_stats stats; 107 108 /* Receive */ 109 struct list_head psock_ready_list; 110 struct bpf_prog *bpf_prog; 111 struct kcm_sock *rx_kcm; 112 unsigned long long saved_rx_bytes; 113 unsigned long long saved_rx_msgs; 114 struct sk_buff *ready_rx_msg; 115 116 /* Transmit */ 117 struct kcm_sock *tx_kcm; 118 struct list_head psock_avail_list; 119 unsigned long long saved_tx_bytes; 120 unsigned long long saved_tx_msgs; 121 }; 122 123 /* Per net MUX list */ 124 struct kcm_net { 125 struct mutex mutex; 126 struct kcm_psock_stats aggregate_psock_stats; 127 struct kcm_mux_stats aggregate_mux_stats; 128 struct strp_aggr_stats aggregate_strp_stats; 129 struct list_head mux_list; 130 int count; 131 }; 132 133 /* Structure for a MUX */ 134 struct kcm_mux { 135 struct list_head kcm_mux_list; 136 struct rcu_head rcu; 137 struct kcm_net *knet; 138 139 struct list_head kcm_socks; /* All KCM sockets on MUX */ 140 int kcm_socks_cnt; /* Total KCM socket count for MUX */ 141 struct list_head psocks; /* List of all psocks on MUX */ 142 int psocks_cnt; /* Total attached sockets */ 143 144 struct kcm_mux_stats stats; 145 struct kcm_psock_stats aggregate_psock_stats; 146 struct strp_aggr_stats aggregate_strp_stats; 147 148 /* Receive */ 149 spinlock_t rx_lock ____cacheline_aligned_in_smp; 150 struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */ 151 struct list_head psocks_ready; /* List of psocks with a msg ready */ 152 struct sk_buff_head rx_hold_queue; 153 154 /* Transmit */ 155 spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */ 156 struct list_head psocks_avail; /* List of available psocks */ 157 struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */ 158 }; 159 160 #ifdef CONFIG_PROC_FS 161 int kcm_proc_init(void); 162 void kcm_proc_exit(void); 163 #else kcm_proc_init(void)164static inline int kcm_proc_init(void) { return 0; } kcm_proc_exit(void)165static inline void kcm_proc_exit(void) { } 166 #endif 167 aggregate_psock_stats(struct kcm_psock_stats * stats,struct kcm_psock_stats * agg_stats)168static inline void aggregate_psock_stats(struct kcm_psock_stats *stats, 169 struct kcm_psock_stats *agg_stats) 170 { 171 /* Save psock statistics in the mux when psock is being unattached. */ 172 173 #define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat) 174 SAVE_PSOCK_STATS(tx_msgs); 175 SAVE_PSOCK_STATS(tx_bytes); 176 SAVE_PSOCK_STATS(reserved); 177 SAVE_PSOCK_STATS(unreserved); 178 SAVE_PSOCK_STATS(tx_aborts); 179 #undef SAVE_PSOCK_STATS 180 } 181 aggregate_mux_stats(struct kcm_mux_stats * stats,struct kcm_mux_stats * agg_stats)182static inline void aggregate_mux_stats(struct kcm_mux_stats *stats, 183 struct kcm_mux_stats *agg_stats) 184 { 185 /* Save psock statistics in the mux when psock is being unattached. */ 186 187 #define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat) 188 SAVE_MUX_STATS(rx_msgs); 189 SAVE_MUX_STATS(rx_bytes); 190 SAVE_MUX_STATS(tx_msgs); 191 SAVE_MUX_STATS(tx_bytes); 192 SAVE_MUX_STATS(rx_ready_drops); 193 SAVE_MUX_STATS(psock_attach); 194 SAVE_MUX_STATS(psock_unattach_rsvd); 195 SAVE_MUX_STATS(psock_unattach); 196 #undef SAVE_MUX_STATS 197 } 198 199 #endif /* __NET_KCM_H_ */ 200