1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Connection Data Control (CDC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 10 */ 11 12 #ifndef SMC_CDC_H 13 #define SMC_CDC_H 14 15 #include <linux/kernel.h> /* max_t */ 16 #include <linux/atomic.h> 17 #include <linux/in.h> 18 #include <linux/compiler.h> 19 20 #include "smc.h" 21 #include "smc_core.h" 22 #include "smc_wr.h" 23 24 #define SMC_CDC_MSG_TYPE 0xFE 25 26 /* in network byte order */ 27 union smc_cdc_cursor { /* SMC cursor */ 28 struct { 29 __be16 reserved; 30 __be16 wrap; 31 __be32 count; 32 }; 33 #ifdef KERNEL_HAS_ATOMIC64 34 atomic64_t acurs; /* for atomic processing */ 35 #else 36 u64 acurs; /* for atomic processing */ 37 #endif 38 } __aligned(8); 39 40 /* in network byte order */ 41 struct smc_cdc_msg { 42 struct smc_wr_rx_hdr common; /* .type = 0xFE */ 43 u8 len; /* 44 */ 44 __be16 seqno; 45 __be32 token; 46 union smc_cdc_cursor prod; 47 union smc_cdc_cursor cons; /* piggy backed "ack" */ 48 struct smc_cdc_producer_flags prod_flags; 49 struct smc_cdc_conn_state_flags conn_state_flags; 50 u8 reserved[18]; 51 }; 52 53 /* SMC-D cursor format */ 54 union smcd_cdc_cursor { 55 struct { 56 u16 wrap; 57 u32 count; 58 struct smc_cdc_producer_flags prod_flags; 59 struct smc_cdc_conn_state_flags conn_state_flags; 60 } __packed; 61 #ifdef KERNEL_HAS_ATOMIC64 62 atomic64_t acurs; /* for atomic processing */ 63 #else 64 u64 acurs; /* for atomic processing */ 65 #endif 66 } __aligned(8); 67 68 /* CDC message for SMC-D */ 69 struct smcd_cdc_msg { 70 struct smc_wr_rx_hdr common; /* Type = 0xFE */ 71 u8 res1[7]; 72 union smcd_cdc_cursor prod; 73 union smcd_cdc_cursor cons; 74 u8 res3[8]; 75 } __aligned(8); 76 77 static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn) 78 { 79 return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort || 80 conn->local_rx_ctrl.conn_state_flags.peer_conn_closed; 81 } 82 83 static inline bool smc_cdc_rxed_any_close_or_senddone( 84 struct smc_connection *conn) 85 { 86 return smc_cdc_rxed_any_close(conn) || 87 conn->local_rx_ctrl.conn_state_flags.peer_done_writing; 88 } 89 90 static inline void smc_curs_add(int size, union smc_host_cursor *curs, 91 int value) 92 { 93 curs->count += value; 94 if (curs->count >= size) { 95 curs->wrap++; 96 curs->count -= size; 97 } 98 } 99 100 /* SMC cursors are 8 bytes long and require atomic reading and writing */ 101 static inline u64 smc_curs_read(union smc_host_cursor *curs, 102 struct smc_connection *conn) 103 { 104 #ifndef KERNEL_HAS_ATOMIC64 105 unsigned long flags; 106 u64 ret; 107 108 spin_lock_irqsave(&conn->acurs_lock, flags); 109 ret = curs->acurs; 110 spin_unlock_irqrestore(&conn->acurs_lock, flags); 111 return ret; 112 #else 113 return atomic64_read(&curs->acurs); 114 #endif 115 } 116 117 /* Copy cursor src into tgt */ 118 static inline void smc_curs_copy(union smc_host_cursor *tgt, 119 union smc_host_cursor *src, 120 struct smc_connection *conn) 121 { 122 #ifndef KERNEL_HAS_ATOMIC64 123 unsigned long flags; 124 125 spin_lock_irqsave(&conn->acurs_lock, flags); 126 tgt->acurs = src->acurs; 127 spin_unlock_irqrestore(&conn->acurs_lock, flags); 128 #else 129 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs)); 130 #endif 131 } 132 133 static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt, 134 union smc_cdc_cursor *src, 135 struct smc_connection *conn) 136 { 137 #ifndef KERNEL_HAS_ATOMIC64 138 unsigned long flags; 139 140 spin_lock_irqsave(&conn->acurs_lock, flags); 141 tgt->acurs = src->acurs; 142 spin_unlock_irqrestore(&conn->acurs_lock, flags); 143 #else 144 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs)); 145 #endif 146 } 147 148 static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt, 149 union smcd_cdc_cursor *src, 150 struct smc_connection *conn) 151 { 152 #ifndef KERNEL_HAS_ATOMIC64 153 unsigned long flags; 154 155 spin_lock_irqsave(&conn->acurs_lock, flags); 156 tgt->acurs = src->acurs; 157 spin_unlock_irqrestore(&conn->acurs_lock, flags); 158 #else 159 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs)); 160 #endif 161 } 162 163 /* calculate cursor difference between old and new, where old <= new and 164 * difference cannot exceed size 165 */ 166 static inline int smc_curs_diff(unsigned int size, 167 union smc_host_cursor *old, 168 union smc_host_cursor *new) 169 { 170 if (old->wrap != new->wrap) 171 return max_t(int, 0, 172 ((size - old->count) + new->count)); 173 174 return max_t(int, 0, (new->count - old->count)); 175 } 176 177 /* calculate cursor difference between old and new - returns negative 178 * value in case old > new 179 */ 180 static inline int smc_curs_comp(unsigned int size, 181 union smc_host_cursor *old, 182 union smc_host_cursor *new) 183 { 184 if (old->wrap > new->wrap || 185 (old->wrap == new->wrap && old->count > new->count)) 186 return -smc_curs_diff(size, new, old); 187 return smc_curs_diff(size, old, new); 188 } 189 190 /* calculate cursor difference between old and new, where old <= new and 191 * difference may exceed size 192 */ 193 static inline int smc_curs_diff_large(unsigned int size, 194 union smc_host_cursor *old, 195 union smc_host_cursor *new) 196 { 197 if (old->wrap < new->wrap) 198 return min_t(int, 199 (size - old->count) + new->count + 200 (new->wrap - old->wrap - 1) * size, 201 size); 202 203 if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */ 204 return min_t(int, 205 (size - old->count) + new->count + 206 (new->wrap + 0xffff - old->wrap) * size, 207 size); 208 209 return max_t(int, 0, (new->count - old->count)); 210 } 211 212 static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer, 213 union smc_host_cursor *local, 214 union smc_host_cursor *save, 215 struct smc_connection *conn) 216 { 217 smc_curs_copy(save, local, conn); 218 peer->count = htonl(save->count); 219 peer->wrap = htons(save->wrap); 220 /* peer->reserved = htons(0); must be ensured by caller */ 221 } 222 223 static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer, 224 struct smc_connection *conn, 225 union smc_host_cursor *save) 226 { 227 struct smc_host_cdc_msg *local = &conn->local_tx_ctrl; 228 229 peer->common.type = local->common.type; 230 peer->len = local->len; 231 peer->seqno = htons(local->seqno); 232 peer->token = htonl(local->token); 233 smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn); 234 smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn); 235 peer->prod_flags = local->prod_flags; 236 peer->conn_state_flags = local->conn_state_flags; 237 } 238 239 static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local, 240 union smc_cdc_cursor *peer, 241 struct smc_connection *conn) 242 { 243 union smc_host_cursor temp, old; 244 union smc_cdc_cursor net; 245 246 smc_curs_copy(&old, local, conn); 247 smc_curs_copy_net(&net, peer, conn); 248 temp.count = ntohl(net.count); 249 temp.wrap = ntohs(net.wrap); 250 if ((old.wrap > temp.wrap) && temp.wrap) 251 return; 252 if ((old.wrap == temp.wrap) && 253 (old.count > temp.count)) 254 return; 255 smc_curs_copy(local, &temp, conn); 256 } 257 258 static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local, 259 struct smc_cdc_msg *peer, 260 struct smc_connection *conn) 261 { 262 local->common.type = peer->common.type; 263 local->len = peer->len; 264 local->seqno = ntohs(peer->seqno); 265 local->token = ntohl(peer->token); 266 smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn); 267 smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn); 268 local->prod_flags = peer->prod_flags; 269 local->conn_state_flags = peer->conn_state_flags; 270 } 271 272 static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local, 273 struct smcd_cdc_msg *peer, 274 struct smc_connection *conn) 275 { 276 union smc_host_cursor temp; 277 278 temp.wrap = peer->prod.wrap; 279 temp.count = peer->prod.count; 280 smc_curs_copy(&local->prod, &temp, conn); 281 282 temp.wrap = peer->cons.wrap; 283 temp.count = peer->cons.count; 284 smc_curs_copy(&local->cons, &temp, conn); 285 local->prod_flags = peer->cons.prod_flags; 286 local->conn_state_flags = peer->cons.conn_state_flags; 287 } 288 289 static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local, 290 struct smc_cdc_msg *peer, 291 struct smc_connection *conn) 292 { 293 if (conn->lgr->is_smcd) 294 smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn); 295 else 296 smcr_cdc_msg_to_host(local, peer, conn); 297 } 298 299 struct smc_cdc_tx_pend { 300 struct smc_connection *conn; /* socket connection */ 301 union smc_host_cursor cursor; /* tx sndbuf cursor sent */ 302 union smc_host_cursor p_cursor; /* rx RMBE cursor produced */ 303 u16 ctrl_seq; /* conn. tx sequence # */ 304 }; 305 306 int smc_cdc_get_free_slot(struct smc_connection *conn, 307 struct smc_wr_buf **wr_buf, 308 struct smc_rdma_wr **wr_rdma_buf, 309 struct smc_cdc_tx_pend **pend); 310 void smc_cdc_tx_dismiss_slots(struct smc_connection *conn); 311 int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf, 312 struct smc_cdc_tx_pend *pend); 313 int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn); 314 int smcd_cdc_msg_send(struct smc_connection *conn); 315 int smc_cdc_init(void) __init; 316 void smcd_cdc_rx_init(struct smc_connection *conn); 317 318 #endif /* SMC_CDC_H */ 319