1 /* 2 * Kernel Connection Multiplexor 3 * 4 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 8 * as published by the Free Software Foundation. 9 */ 10 11 #ifndef __NET_KCM_H_ 12 #define __NET_KCM_H_ 13 14 #include <linux/skbuff.h> 15 #include <net/sock.h> 16 #include <net/strparser.h> 17 #include <uapi/linux/kcm.h> 18 19 extern unsigned int kcm_net_id; 20 21 #define KCM_STATS_ADD(stat, count) ((stat) += (count)) 22 #define KCM_STATS_INCR(stat) ((stat)++) 23 24 struct kcm_psock_stats { 25 unsigned long long tx_msgs; 26 unsigned long long tx_bytes; 27 unsigned long long reserved; 28 unsigned long long unreserved; 29 unsigned int tx_aborts; 30 }; 31 32 struct kcm_mux_stats { 33 unsigned long long rx_msgs; 34 unsigned long long rx_bytes; 35 unsigned long long tx_msgs; 36 unsigned long long tx_bytes; 37 unsigned int rx_ready_drops; 38 unsigned int tx_retries; 39 unsigned int psock_attach; 40 unsigned int psock_unattach_rsvd; 41 unsigned int psock_unattach; 42 }; 43 44 struct kcm_stats { 45 unsigned long long rx_msgs; 46 unsigned long long rx_bytes; 47 unsigned long long tx_msgs; 48 unsigned long long tx_bytes; 49 }; 50 51 struct kcm_tx_msg { 52 unsigned int sent; 53 unsigned int fragidx; 54 unsigned int frag_offset; 55 unsigned int msg_flags; 56 struct sk_buff *frag_skb; 57 struct sk_buff *last_skb; 58 }; 59 60 /* Socket structure for KCM client sockets */ 61 struct kcm_sock { 62 struct sock sk; 63 struct kcm_mux *mux; 64 struct list_head kcm_sock_list; 65 int index; 66 u32 done : 1; 67 struct work_struct done_work; 68 69 struct kcm_stats stats; 70 71 /* Transmit */ 72 struct kcm_psock *tx_psock; 73 struct work_struct tx_work; 74 struct list_head wait_psock_list; 75 struct sk_buff *seq_skb; 76 u32 tx_stopped : 1; 77 78 /* Don't use bit fields here, these are set under different locks */ 79 bool tx_wait; 80 bool tx_wait_more; 81 82 /* Receive */ 83 struct kcm_psock *rx_psock; 84 struct list_head wait_rx_list; /* KCMs waiting for receiving */ 85 bool rx_wait; 86 u32 rx_disabled : 1; 87 }; 88 89 struct bpf_prog; 90 91 /* Structure for an attached lower socket */ 92 struct kcm_psock { 93 struct sock *sk; 94 struct strparser strp; 95 struct kcm_mux *mux; 96 int index; 97 98 u32 tx_stopped : 1; 99 u32 done : 1; 100 u32 unattaching : 1; 101 102 void (*save_state_change)(struct sock *sk); 103 void (*save_data_ready)(struct sock *sk); 104 void (*save_write_space)(struct sock *sk); 105 106 struct list_head psock_list; 107 108 struct kcm_psock_stats stats; 109 110 /* Receive */ 111 struct list_head psock_ready_list; 112 struct bpf_prog *bpf_prog; 113 struct kcm_sock *rx_kcm; 114 unsigned long long saved_rx_bytes; 115 unsigned long long saved_rx_msgs; 116 struct sk_buff *ready_rx_msg; 117 118 /* Transmit */ 119 struct kcm_sock *tx_kcm; 120 struct list_head psock_avail_list; 121 unsigned long long saved_tx_bytes; 122 unsigned long long saved_tx_msgs; 123 }; 124 125 /* Per net MUX list */ 126 struct kcm_net { 127 struct mutex mutex; 128 struct kcm_psock_stats aggregate_psock_stats; 129 struct kcm_mux_stats aggregate_mux_stats; 130 struct strp_aggr_stats aggregate_strp_stats; 131 struct list_head mux_list; 132 int count; 133 }; 134 135 /* Structure for a MUX */ 136 struct kcm_mux { 137 struct list_head kcm_mux_list; 138 struct rcu_head rcu; 139 struct kcm_net *knet; 140 141 struct list_head kcm_socks; /* All KCM sockets on MUX */ 142 int kcm_socks_cnt; /* Total KCM socket count for MUX */ 143 struct list_head psocks; /* List of all psocks on MUX */ 144 int psocks_cnt; /* Total attached sockets */ 145 146 struct kcm_mux_stats stats; 147 struct kcm_psock_stats aggregate_psock_stats; 148 struct strp_aggr_stats aggregate_strp_stats; 149 150 /* Receive */ 151 spinlock_t rx_lock ____cacheline_aligned_in_smp; 152 struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */ 153 struct list_head psocks_ready; /* List of psocks with a msg ready */ 154 struct sk_buff_head rx_hold_queue; 155 156 /* Transmit */ 157 spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */ 158 struct list_head psocks_avail; /* List of available psocks */ 159 struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */ 160 }; 161 162 #ifdef CONFIG_PROC_FS 163 int kcm_proc_init(void); 164 void kcm_proc_exit(void); 165 #else 166 static inline int kcm_proc_init(void) { return 0; } 167 static inline void kcm_proc_exit(void) { } 168 #endif 169 170 static inline void aggregate_psock_stats(struct kcm_psock_stats *stats, 171 struct kcm_psock_stats *agg_stats) 172 { 173 /* Save psock statistics in the mux when psock is being unattached. */ 174 175 #define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat) 176 SAVE_PSOCK_STATS(tx_msgs); 177 SAVE_PSOCK_STATS(tx_bytes); 178 SAVE_PSOCK_STATS(reserved); 179 SAVE_PSOCK_STATS(unreserved); 180 SAVE_PSOCK_STATS(tx_aborts); 181 #undef SAVE_PSOCK_STATS 182 } 183 184 static inline void aggregate_mux_stats(struct kcm_mux_stats *stats, 185 struct kcm_mux_stats *agg_stats) 186 { 187 /* Save psock statistics in the mux when psock is being unattached. */ 188 189 #define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat) 190 SAVE_MUX_STATS(rx_msgs); 191 SAVE_MUX_STATS(rx_bytes); 192 SAVE_MUX_STATS(tx_msgs); 193 SAVE_MUX_STATS(tx_bytes); 194 SAVE_MUX_STATS(rx_ready_drops); 195 SAVE_MUX_STATS(psock_attach); 196 SAVE_MUX_STATS(psock_unattach_rsvd); 197 SAVE_MUX_STATS(psock_unattach); 198 #undef SAVE_MUX_STATS 199 } 200 201 #endif /* __NET_KCM_H_ */ 202