1 /* 2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #ifndef RXE_VERBS_H 35 #define RXE_VERBS_H 36 37 #include <linux/interrupt.h> 38 #include <rdma/rdma_user_rxe.h> 39 #include "rxe_pool.h" 40 #include "rxe_task.h" 41 #include "rxe_hw_counters.h" 42 43 static inline int pkey_match(u16 key1, u16 key2) 44 { 45 return (((key1 & 0x7fff) != 0) && 46 ((key1 & 0x7fff) == (key2 & 0x7fff)) && 47 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0; 48 } 49 50 /* Return >0 if psn_a > psn_b 51 * 0 if psn_a == psn_b 52 * <0 if psn_a < psn_b 53 */ 54 static inline int psn_compare(u32 psn_a, u32 psn_b) 55 { 56 s32 diff; 57 58 diff = (psn_a - psn_b) << 8; 59 return diff; 60 } 61 62 struct rxe_ucontext { 63 struct rxe_pool_entry pelem; 64 struct ib_ucontext ibuc; 65 }; 66 67 struct rxe_pd { 68 struct rxe_pool_entry pelem; 69 struct ib_pd ibpd; 70 }; 71 72 struct rxe_ah { 73 struct rxe_pool_entry pelem; 74 struct ib_ah ibah; 75 struct rxe_pd *pd; 76 struct rxe_av av; 77 }; 78 79 struct rxe_cqe { 80 union { 81 struct ib_wc ibwc; 82 struct ib_uverbs_wc uibwc; 83 }; 84 }; 85 86 struct rxe_cq { 87 struct rxe_pool_entry pelem; 88 struct ib_cq ibcq; 89 struct rxe_queue *queue; 90 spinlock_t cq_lock; 91 u8 notify; 92 bool is_dying; 93 int is_user; 94 struct tasklet_struct comp_task; 95 }; 96 97 enum wqe_state { 98 wqe_state_posted, 99 wqe_state_processing, 100 wqe_state_pending, 101 wqe_state_done, 102 wqe_state_error, 103 }; 104 105 struct rxe_sq { 106 int max_wr; 107 int max_sge; 108 int max_inline; 109 spinlock_t sq_lock; /* guard queue */ 110 struct rxe_queue *queue; 111 }; 112 113 struct rxe_rq { 114 int max_wr; 115 int max_sge; 116 spinlock_t producer_lock; /* guard queue producer */ 117 spinlock_t consumer_lock; /* guard queue consumer */ 118 struct rxe_queue *queue; 119 }; 120 121 struct rxe_srq { 122 struct rxe_pool_entry pelem; 123 struct ib_srq ibsrq; 124 struct rxe_pd *pd; 125 struct rxe_rq rq; 126 u32 srq_num; 127 128 int limit; 129 int error; 130 }; 131 132 enum rxe_qp_state { 133 QP_STATE_RESET, 134 QP_STATE_INIT, 135 QP_STATE_READY, 136 QP_STATE_DRAIN, /* req only */ 137 QP_STATE_DRAINED, /* req only */ 138 QP_STATE_ERROR 139 }; 140 141 extern char *rxe_qp_state_name[]; 142 143 struct rxe_req_info { 144 enum rxe_qp_state state; 145 int wqe_index; 146 u32 psn; 147 int opcode; 148 atomic_t rd_atomic; 149 int wait_fence; 150 int need_rd_atomic; 151 int wait_psn; 152 int need_retry; 153 int noack_pkts; 154 struct rxe_task task; 155 }; 156 157 struct rxe_comp_info { 158 u32 psn; 159 int opcode; 160 int timeout; 161 int timeout_retry; 162 u32 retry_cnt; 163 u32 rnr_retry; 164 struct rxe_task task; 165 }; 166 167 enum rdatm_res_state { 168 rdatm_res_state_next, 169 rdatm_res_state_new, 170 rdatm_res_state_replay, 171 }; 172 173 struct resp_res { 174 int type; 175 u32 first_psn; 176 u32 last_psn; 177 u32 cur_psn; 178 enum rdatm_res_state state; 179 180 union { 181 struct { 182 struct sk_buff *skb; 183 } atomic; 184 struct { 185 struct rxe_mem *mr; 186 u64 va_org; 187 u32 rkey; 188 u32 length; 189 u64 va; 190 u32 resid; 191 } read; 192 }; 193 }; 194 195 struct rxe_resp_info { 196 enum rxe_qp_state state; 197 u32 msn; 198 u32 psn; 199 int opcode; 200 int drop_msg; 201 int goto_error; 202 int sent_psn_nak; 203 enum ib_wc_status status; 204 u8 aeth_syndrome; 205 206 /* Receive only */ 207 struct rxe_recv_wqe *wqe; 208 209 /* RDMA read / atomic only */ 210 u64 va; 211 struct rxe_mem *mr; 212 u32 resid; 213 u32 rkey; 214 u64 atomic_orig; 215 216 /* SRQ only */ 217 struct { 218 struct rxe_recv_wqe wqe; 219 struct ib_sge sge[RXE_MAX_SGE]; 220 } srq_wqe; 221 222 /* Responder resources. It's a circular list where the oldest 223 * resource is dropped first. 224 */ 225 struct resp_res *resources; 226 unsigned int res_head; 227 unsigned int res_tail; 228 struct resp_res *res; 229 struct rxe_task task; 230 }; 231 232 struct rxe_qp { 233 struct rxe_pool_entry pelem; 234 struct ib_qp ibqp; 235 struct ib_qp_attr attr; 236 unsigned int valid; 237 unsigned int mtu; 238 int is_user; 239 240 struct rxe_pd *pd; 241 struct rxe_srq *srq; 242 struct rxe_cq *scq; 243 struct rxe_cq *rcq; 244 245 enum ib_sig_type sq_sig_type; 246 247 struct rxe_sq sq; 248 struct rxe_rq rq; 249 250 struct socket *sk; 251 u32 dst_cookie; 252 253 struct rxe_av pri_av; 254 struct rxe_av alt_av; 255 256 /* list of mcast groups qp has joined (for cleanup) */ 257 struct list_head grp_list; 258 spinlock_t grp_lock; /* guard grp_list */ 259 260 struct sk_buff_head req_pkts; 261 struct sk_buff_head resp_pkts; 262 struct sk_buff_head send_pkts; 263 264 struct rxe_req_info req; 265 struct rxe_comp_info comp; 266 struct rxe_resp_info resp; 267 268 atomic_t ssn; 269 atomic_t skb_out; 270 int need_req_skb; 271 272 /* Timer for retranmitting packet when ACKs have been lost. RC 273 * only. The requester sets it when it is not already 274 * started. The responder resets it whenever an ack is 275 * received. 276 */ 277 struct timer_list retrans_timer; 278 u64 qp_timeout_jiffies; 279 280 /* Timer for handling RNR NAKS. */ 281 struct timer_list rnr_nak_timer; 282 283 spinlock_t state_lock; /* guard requester and completer */ 284 }; 285 286 enum rxe_mem_state { 287 RXE_MEM_STATE_ZOMBIE, 288 RXE_MEM_STATE_INVALID, 289 RXE_MEM_STATE_FREE, 290 RXE_MEM_STATE_VALID, 291 }; 292 293 enum rxe_mem_type { 294 RXE_MEM_TYPE_NONE, 295 RXE_MEM_TYPE_DMA, 296 RXE_MEM_TYPE_MR, 297 RXE_MEM_TYPE_FMR, 298 RXE_MEM_TYPE_MW, 299 }; 300 301 #define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) 302 303 struct rxe_phys_buf { 304 u64 addr; 305 u64 size; 306 }; 307 308 struct rxe_map { 309 struct rxe_phys_buf buf[RXE_BUF_PER_MAP]; 310 }; 311 312 struct rxe_mem { 313 struct rxe_pool_entry pelem; 314 union { 315 struct ib_mr ibmr; 316 struct ib_mw ibmw; 317 }; 318 319 struct rxe_pd *pd; 320 struct ib_umem *umem; 321 322 u32 lkey; 323 u32 rkey; 324 325 enum rxe_mem_state state; 326 enum rxe_mem_type type; 327 u64 va; 328 u64 iova; 329 size_t length; 330 u32 offset; 331 int access; 332 333 int page_shift; 334 int page_mask; 335 int map_shift; 336 int map_mask; 337 338 u32 num_buf; 339 u32 nbuf; 340 341 u32 max_buf; 342 u32 num_map; 343 344 struct rxe_map **map; 345 }; 346 347 struct rxe_mc_grp { 348 struct rxe_pool_entry pelem; 349 spinlock_t mcg_lock; /* guard group */ 350 struct rxe_dev *rxe; 351 struct list_head qp_list; 352 union ib_gid mgid; 353 int num_qp; 354 u32 qkey; 355 u16 pkey; 356 }; 357 358 struct rxe_mc_elem { 359 struct rxe_pool_entry pelem; 360 struct list_head qp_list; 361 struct list_head grp_list; 362 struct rxe_qp *qp; 363 struct rxe_mc_grp *grp; 364 }; 365 366 struct rxe_port { 367 struct ib_port_attr attr; 368 u16 *pkey_tbl; 369 __be64 port_guid; 370 __be64 subnet_prefix; 371 spinlock_t port_lock; /* guard port */ 372 unsigned int mtu_cap; 373 /* special QPs */ 374 u32 qp_smi_index; 375 u32 qp_gsi_index; 376 }; 377 378 struct rxe_dev { 379 struct ib_device ib_dev; 380 struct ib_device_attr attr; 381 int max_ucontext; 382 int max_inline_data; 383 struct kref ref_cnt; 384 struct mutex usdev_lock; 385 386 struct net_device *ndev; 387 388 int xmit_errors; 389 390 struct rxe_pool uc_pool; 391 struct rxe_pool pd_pool; 392 struct rxe_pool ah_pool; 393 struct rxe_pool srq_pool; 394 struct rxe_pool qp_pool; 395 struct rxe_pool cq_pool; 396 struct rxe_pool mr_pool; 397 struct rxe_pool mw_pool; 398 struct rxe_pool mc_grp_pool; 399 struct rxe_pool mc_elem_pool; 400 401 spinlock_t pending_lock; /* guard pending_mmaps */ 402 struct list_head pending_mmaps; 403 404 spinlock_t mmap_offset_lock; /* guard mmap_offset */ 405 int mmap_offset; 406 407 u64 stats_counters[RXE_NUM_OF_COUNTERS]; 408 409 struct rxe_port port; 410 struct list_head list; 411 struct crypto_shash *tfm; 412 }; 413 414 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters cnt) 415 { 416 rxe->stats_counters[cnt]++; 417 } 418 419 static inline struct rxe_dev *to_rdev(struct ib_device *dev) 420 { 421 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL; 422 } 423 424 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc) 425 { 426 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL; 427 } 428 429 static inline struct rxe_pd *to_rpd(struct ib_pd *pd) 430 { 431 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL; 432 } 433 434 static inline struct rxe_ah *to_rah(struct ib_ah *ah) 435 { 436 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL; 437 } 438 439 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq) 440 { 441 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL; 442 } 443 444 static inline struct rxe_qp *to_rqp(struct ib_qp *qp) 445 { 446 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL; 447 } 448 449 static inline struct rxe_cq *to_rcq(struct ib_cq *cq) 450 { 451 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL; 452 } 453 454 static inline struct rxe_mem *to_rmr(struct ib_mr *mr) 455 { 456 return mr ? container_of(mr, struct rxe_mem, ibmr) : NULL; 457 } 458 459 static inline struct rxe_mem *to_rmw(struct ib_mw *mw) 460 { 461 return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL; 462 } 463 464 int rxe_register_device(struct rxe_dev *rxe); 465 int rxe_unregister_device(struct rxe_dev *rxe); 466 467 void rxe_mc_cleanup(struct rxe_pool_entry *arg); 468 469 #endif /* RXE_VERBS_H */ 470