1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Connection Data Control (CDC) 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 10 */ 11 12 #ifndef SMC_CDC_H 13 #define SMC_CDC_H 14 15 #include <linux/kernel.h> /* max_t */ 16 #include <linux/atomic.h> 17 #include <linux/in.h> 18 #include <linux/compiler.h> 19 20 #include "smc.h" 21 #include "smc_core.h" 22 #include "smc_wr.h" 23 24 #define SMC_CDC_MSG_TYPE 0xFE 25 26 /* in network byte order */ 27 union smc_cdc_cursor { /* SMC cursor */ 28 struct { 29 __be16 reserved; 30 __be16 wrap; 31 __be32 count; 32 }; 33 #ifdef KERNEL_HAS_ATOMIC64 34 atomic64_t acurs; /* for atomic processing */ 35 #else 36 u64 acurs; /* for atomic processing */ 37 #endif 38 } __aligned(8); 39 40 /* in network byte order */ 41 struct smc_cdc_msg { 42 struct smc_wr_rx_hdr common; /* .type = 0xFE */ 43 u8 len; /* 44 */ 44 __be16 seqno; 45 __be32 token; 46 union smc_cdc_cursor prod; 47 union smc_cdc_cursor cons; /* piggy backed "ack" */ 48 struct smc_cdc_producer_flags prod_flags; 49 struct smc_cdc_conn_state_flags conn_state_flags; 50 u8 reserved[18]; 51 } __packed; /* format defined in RFC7609 */ 52 53 /* CDC message for SMC-D */ 54 struct smcd_cdc_msg { 55 struct smc_wr_rx_hdr common; /* Type = 0xFE */ 56 u8 res1[7]; 57 u16 prod_wrap; 58 u32 prod_count; 59 u8 res2[2]; 60 u16 cons_wrap; 61 u32 cons_count; 62 struct smc_cdc_producer_flags prod_flags; 63 struct smc_cdc_conn_state_flags conn_state_flags; 64 u8 res3[8]; 65 } __packed; 66 67 static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn) 68 { 69 return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort || 70 conn->local_rx_ctrl.conn_state_flags.peer_conn_closed; 71 } 72 73 static inline bool smc_cdc_rxed_any_close_or_senddone( 74 struct smc_connection *conn) 75 { 76 return smc_cdc_rxed_any_close(conn) || 77 conn->local_rx_ctrl.conn_state_flags.peer_done_writing; 78 } 79 80 static inline void smc_curs_add(int size, union smc_host_cursor *curs, 81 int value) 82 { 83 curs->count += value; 84 if (curs->count >= size) { 85 curs->wrap++; 86 curs->count -= size; 87 } 88 } 89 90 /* SMC cursors are 8 bytes long and require atomic reading and writing */ 91 static inline u64 smc_curs_read(union smc_host_cursor *curs, 92 struct smc_connection *conn) 93 { 94 #ifndef KERNEL_HAS_ATOMIC64 95 unsigned long flags; 96 u64 ret; 97 98 spin_lock_irqsave(&conn->acurs_lock, flags); 99 ret = curs->acurs; 100 spin_unlock_irqrestore(&conn->acurs_lock, flags); 101 return ret; 102 #else 103 return atomic64_read(&curs->acurs); 104 #endif 105 } 106 107 /* Copy cursor src into tgt */ 108 static inline void smc_curs_copy(union smc_host_cursor *tgt, 109 union smc_host_cursor *src, 110 struct smc_connection *conn) 111 { 112 #ifndef KERNEL_HAS_ATOMIC64 113 unsigned long flags; 114 115 spin_lock_irqsave(&conn->acurs_lock, flags); 116 tgt->acurs = src->acurs; 117 spin_unlock_irqrestore(&conn->acurs_lock, flags); 118 #else 119 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs)); 120 #endif 121 } 122 123 static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt, 124 union smc_cdc_cursor *src, 125 struct smc_connection *conn) 126 { 127 #ifndef KERNEL_HAS_ATOMIC64 128 unsigned long flags; 129 130 spin_lock_irqsave(&conn->acurs_lock, flags); 131 tgt->acurs = src->acurs; 132 spin_unlock_irqrestore(&conn->acurs_lock, flags); 133 #else 134 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs)); 135 #endif 136 } 137 138 /* calculate cursor difference between old and new, where old <= new */ 139 static inline int smc_curs_diff(unsigned int size, 140 union smc_host_cursor *old, 141 union smc_host_cursor *new) 142 { 143 if (old->wrap != new->wrap) 144 return max_t(int, 0, 145 ((size - old->count) + new->count)); 146 147 return max_t(int, 0, (new->count - old->count)); 148 } 149 150 /* calculate cursor difference between old and new - returns negative 151 * value in case old > new 152 */ 153 static inline int smc_curs_comp(unsigned int size, 154 union smc_host_cursor *old, 155 union smc_host_cursor *new) 156 { 157 if (old->wrap > new->wrap || 158 (old->wrap == new->wrap && old->count > new->count)) 159 return -smc_curs_diff(size, new, old); 160 return smc_curs_diff(size, old, new); 161 } 162 163 static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer, 164 union smc_host_cursor *local, 165 struct smc_connection *conn) 166 { 167 union smc_host_cursor temp; 168 169 smc_curs_copy(&temp, local, conn); 170 peer->count = htonl(temp.count); 171 peer->wrap = htons(temp.wrap); 172 /* peer->reserved = htons(0); must be ensured by caller */ 173 } 174 175 static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer, 176 struct smc_host_cdc_msg *local, 177 struct smc_connection *conn) 178 { 179 peer->common.type = local->common.type; 180 peer->len = local->len; 181 peer->seqno = htons(local->seqno); 182 peer->token = htonl(local->token); 183 smc_host_cursor_to_cdc(&peer->prod, &local->prod, conn); 184 smc_host_cursor_to_cdc(&peer->cons, &local->cons, conn); 185 peer->prod_flags = local->prod_flags; 186 peer->conn_state_flags = local->conn_state_flags; 187 } 188 189 static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local, 190 union smc_cdc_cursor *peer, 191 struct smc_connection *conn) 192 { 193 union smc_host_cursor temp, old; 194 union smc_cdc_cursor net; 195 196 smc_curs_copy(&old, local, conn); 197 smc_curs_copy_net(&net, peer, conn); 198 temp.count = ntohl(net.count); 199 temp.wrap = ntohs(net.wrap); 200 if ((old.wrap > temp.wrap) && temp.wrap) 201 return; 202 if ((old.wrap == temp.wrap) && 203 (old.count > temp.count)) 204 return; 205 smc_curs_copy(local, &temp, conn); 206 } 207 208 static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local, 209 struct smc_cdc_msg *peer, 210 struct smc_connection *conn) 211 { 212 local->common.type = peer->common.type; 213 local->len = peer->len; 214 local->seqno = ntohs(peer->seqno); 215 local->token = ntohl(peer->token); 216 smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn); 217 smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn); 218 local->prod_flags = peer->prod_flags; 219 local->conn_state_flags = peer->conn_state_flags; 220 } 221 222 static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local, 223 struct smcd_cdc_msg *peer) 224 { 225 local->prod.wrap = peer->prod_wrap; 226 local->prod.count = peer->prod_count; 227 local->cons.wrap = peer->cons_wrap; 228 local->cons.count = peer->cons_count; 229 local->prod_flags = peer->prod_flags; 230 local->conn_state_flags = peer->conn_state_flags; 231 } 232 233 static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local, 234 struct smc_cdc_msg *peer, 235 struct smc_connection *conn) 236 { 237 if (conn->lgr->is_smcd) 238 smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer); 239 else 240 smcr_cdc_msg_to_host(local, peer, conn); 241 } 242 243 struct smc_cdc_tx_pend; 244 245 int smc_cdc_get_free_slot(struct smc_connection *conn, 246 struct smc_wr_buf **wr_buf, 247 struct smc_cdc_tx_pend **pend); 248 void smc_cdc_tx_dismiss_slots(struct smc_connection *conn); 249 int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf, 250 struct smc_cdc_tx_pend *pend); 251 int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn); 252 int smcd_cdc_msg_send(struct smc_connection *conn); 253 int smc_cdc_init(void) __init; 254 void smcd_cdc_rx_init(struct smc_connection *conn); 255 256 #endif /* SMC_CDC_H */ 257