1 /* 2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #ifndef RXE_VERBS_H 35 #define RXE_VERBS_H 36 37 #include <linux/interrupt.h> 38 #include <linux/workqueue.h> 39 #include <rdma/rdma_user_rxe.h> 40 #include "rxe_pool.h" 41 #include "rxe_task.h" 42 #include "rxe_hw_counters.h" 43 44 static inline int pkey_match(u16 key1, u16 key2) 45 { 46 return (((key1 & 0x7fff) != 0) && 47 ((key1 & 0x7fff) == (key2 & 0x7fff)) && 48 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0; 49 } 50 51 /* Return >0 if psn_a > psn_b 52 * 0 if psn_a == psn_b 53 * <0 if psn_a < psn_b 54 */ 55 static inline int psn_compare(u32 psn_a, u32 psn_b) 56 { 57 s32 diff; 58 59 diff = (psn_a - psn_b) << 8; 60 return diff; 61 } 62 63 struct rxe_ucontext { 64 struct rxe_pool_entry pelem; 65 struct ib_ucontext ibuc; 66 }; 67 68 struct rxe_pd { 69 struct rxe_pool_entry pelem; 70 struct ib_pd ibpd; 71 }; 72 73 struct rxe_ah { 74 struct rxe_pool_entry pelem; 75 struct ib_ah ibah; 76 struct rxe_pd *pd; 77 struct rxe_av av; 78 }; 79 80 struct rxe_cqe { 81 union { 82 struct ib_wc ibwc; 83 struct ib_uverbs_wc uibwc; 84 }; 85 }; 86 87 struct rxe_cq { 88 struct rxe_pool_entry pelem; 89 struct ib_cq ibcq; 90 struct rxe_queue *queue; 91 spinlock_t cq_lock; 92 u8 notify; 93 bool is_dying; 94 int is_user; 95 struct tasklet_struct comp_task; 96 }; 97 98 enum wqe_state { 99 wqe_state_posted, 100 wqe_state_processing, 101 wqe_state_pending, 102 wqe_state_done, 103 wqe_state_error, 104 }; 105 106 struct rxe_sq { 107 int max_wr; 108 int max_sge; 109 int max_inline; 110 spinlock_t sq_lock; /* guard queue */ 111 struct rxe_queue *queue; 112 }; 113 114 struct rxe_rq { 115 int max_wr; 116 int max_sge; 117 spinlock_t producer_lock; /* guard queue producer */ 118 spinlock_t consumer_lock; /* guard queue consumer */ 119 struct rxe_queue *queue; 120 }; 121 122 struct rxe_srq { 123 struct rxe_pool_entry pelem; 124 struct ib_srq ibsrq; 125 struct rxe_pd *pd; 126 struct rxe_rq rq; 127 u32 srq_num; 128 129 int limit; 130 int error; 131 }; 132 133 enum rxe_qp_state { 134 QP_STATE_RESET, 135 QP_STATE_INIT, 136 QP_STATE_READY, 137 QP_STATE_DRAIN, /* req only */ 138 QP_STATE_DRAINED, /* req only */ 139 QP_STATE_ERROR 140 }; 141 142 extern char *rxe_qp_state_name[]; 143 144 struct rxe_req_info { 145 enum rxe_qp_state state; 146 int wqe_index; 147 u32 psn; 148 int opcode; 149 atomic_t rd_atomic; 150 int wait_fence; 151 int need_rd_atomic; 152 int wait_psn; 153 int need_retry; 154 int noack_pkts; 155 struct rxe_task task; 156 }; 157 158 struct rxe_comp_info { 159 u32 psn; 160 int opcode; 161 int timeout; 162 int timeout_retry; 163 u32 retry_cnt; 164 u32 rnr_retry; 165 struct rxe_task task; 166 }; 167 168 enum rdatm_res_state { 169 rdatm_res_state_next, 170 rdatm_res_state_new, 171 rdatm_res_state_replay, 172 }; 173 174 struct resp_res { 175 int type; 176 u32 first_psn; 177 u32 last_psn; 178 u32 cur_psn; 179 enum rdatm_res_state state; 180 181 union { 182 struct { 183 struct sk_buff *skb; 184 } atomic; 185 struct { 186 struct rxe_mem *mr; 187 u64 va_org; 188 u32 rkey; 189 u32 length; 190 u64 va; 191 u32 resid; 192 } read; 193 }; 194 }; 195 196 struct rxe_resp_info { 197 enum rxe_qp_state state; 198 u32 msn; 199 u32 psn; 200 int opcode; 201 int drop_msg; 202 int goto_error; 203 int sent_psn_nak; 204 enum ib_wc_status status; 205 u8 aeth_syndrome; 206 207 /* Receive only */ 208 struct rxe_recv_wqe *wqe; 209 210 /* RDMA read / atomic only */ 211 u64 va; 212 struct rxe_mem *mr; 213 u32 resid; 214 u32 rkey; 215 u64 atomic_orig; 216 217 /* SRQ only */ 218 struct { 219 struct rxe_recv_wqe wqe; 220 struct ib_sge sge[RXE_MAX_SGE]; 221 } srq_wqe; 222 223 /* Responder resources. It's a circular list where the oldest 224 * resource is dropped first. 225 */ 226 struct resp_res *resources; 227 unsigned int res_head; 228 unsigned int res_tail; 229 struct resp_res *res; 230 struct rxe_task task; 231 }; 232 233 struct rxe_qp { 234 struct rxe_pool_entry pelem; 235 struct ib_qp ibqp; 236 struct ib_qp_attr attr; 237 unsigned int valid; 238 unsigned int mtu; 239 int is_user; 240 241 struct rxe_pd *pd; 242 struct rxe_srq *srq; 243 struct rxe_cq *scq; 244 struct rxe_cq *rcq; 245 246 enum ib_sig_type sq_sig_type; 247 248 struct rxe_sq sq; 249 struct rxe_rq rq; 250 251 struct socket *sk; 252 u32 dst_cookie; 253 254 struct rxe_av pri_av; 255 struct rxe_av alt_av; 256 257 /* list of mcast groups qp has joined (for cleanup) */ 258 struct list_head grp_list; 259 spinlock_t grp_lock; /* guard grp_list */ 260 261 struct sk_buff_head req_pkts; 262 struct sk_buff_head resp_pkts; 263 struct sk_buff_head send_pkts; 264 265 struct rxe_req_info req; 266 struct rxe_comp_info comp; 267 struct rxe_resp_info resp; 268 269 atomic_t ssn; 270 atomic_t skb_out; 271 int need_req_skb; 272 273 /* Timer for retranmitting packet when ACKs have been lost. RC 274 * only. The requester sets it when it is not already 275 * started. The responder resets it whenever an ack is 276 * received. 277 */ 278 struct timer_list retrans_timer; 279 u64 qp_timeout_jiffies; 280 281 /* Timer for handling RNR NAKS. */ 282 struct timer_list rnr_nak_timer; 283 284 spinlock_t state_lock; /* guard requester and completer */ 285 286 struct execute_work cleanup_work; 287 }; 288 289 enum rxe_mem_state { 290 RXE_MEM_STATE_ZOMBIE, 291 RXE_MEM_STATE_INVALID, 292 RXE_MEM_STATE_FREE, 293 RXE_MEM_STATE_VALID, 294 }; 295 296 enum rxe_mem_type { 297 RXE_MEM_TYPE_NONE, 298 RXE_MEM_TYPE_DMA, 299 RXE_MEM_TYPE_MR, 300 RXE_MEM_TYPE_FMR, 301 RXE_MEM_TYPE_MW, 302 }; 303 304 #define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf)) 305 306 struct rxe_phys_buf { 307 u64 addr; 308 u64 size; 309 }; 310 311 struct rxe_map { 312 struct rxe_phys_buf buf[RXE_BUF_PER_MAP]; 313 }; 314 315 struct rxe_mem { 316 struct rxe_pool_entry pelem; 317 union { 318 struct ib_mr ibmr; 319 struct ib_mw ibmw; 320 }; 321 322 struct rxe_pd *pd; 323 struct ib_umem *umem; 324 325 u32 lkey; 326 u32 rkey; 327 328 enum rxe_mem_state state; 329 enum rxe_mem_type type; 330 u64 va; 331 u64 iova; 332 size_t length; 333 u32 offset; 334 int access; 335 336 int page_shift; 337 int page_mask; 338 int map_shift; 339 int map_mask; 340 341 u32 num_buf; 342 u32 nbuf; 343 344 u32 max_buf; 345 u32 num_map; 346 347 struct rxe_map **map; 348 }; 349 350 struct rxe_mc_grp { 351 struct rxe_pool_entry pelem; 352 spinlock_t mcg_lock; /* guard group */ 353 struct rxe_dev *rxe; 354 struct list_head qp_list; 355 union ib_gid mgid; 356 int num_qp; 357 u32 qkey; 358 u16 pkey; 359 }; 360 361 struct rxe_mc_elem { 362 struct rxe_pool_entry pelem; 363 struct list_head qp_list; 364 struct list_head grp_list; 365 struct rxe_qp *qp; 366 struct rxe_mc_grp *grp; 367 }; 368 369 struct rxe_port { 370 struct ib_port_attr attr; 371 u16 *pkey_tbl; 372 __be64 port_guid; 373 __be64 subnet_prefix; 374 spinlock_t port_lock; /* guard port */ 375 unsigned int mtu_cap; 376 /* special QPs */ 377 u32 qp_smi_index; 378 u32 qp_gsi_index; 379 }; 380 381 struct rxe_dev { 382 struct ib_device ib_dev; 383 struct ib_device_attr attr; 384 int max_ucontext; 385 int max_inline_data; 386 struct kref ref_cnt; 387 struct mutex usdev_lock; 388 389 struct net_device *ndev; 390 391 int xmit_errors; 392 393 struct rxe_pool uc_pool; 394 struct rxe_pool pd_pool; 395 struct rxe_pool ah_pool; 396 struct rxe_pool srq_pool; 397 struct rxe_pool qp_pool; 398 struct rxe_pool cq_pool; 399 struct rxe_pool mr_pool; 400 struct rxe_pool mw_pool; 401 struct rxe_pool mc_grp_pool; 402 struct rxe_pool mc_elem_pool; 403 404 spinlock_t pending_lock; /* guard pending_mmaps */ 405 struct list_head pending_mmaps; 406 407 spinlock_t mmap_offset_lock; /* guard mmap_offset */ 408 int mmap_offset; 409 410 u64 stats_counters[RXE_NUM_OF_COUNTERS]; 411 412 struct rxe_port port; 413 struct list_head list; 414 struct crypto_shash *tfm; 415 }; 416 417 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters cnt) 418 { 419 rxe->stats_counters[cnt]++; 420 } 421 422 static inline struct rxe_dev *to_rdev(struct ib_device *dev) 423 { 424 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL; 425 } 426 427 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc) 428 { 429 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL; 430 } 431 432 static inline struct rxe_pd *to_rpd(struct ib_pd *pd) 433 { 434 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL; 435 } 436 437 static inline struct rxe_ah *to_rah(struct ib_ah *ah) 438 { 439 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL; 440 } 441 442 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq) 443 { 444 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL; 445 } 446 447 static inline struct rxe_qp *to_rqp(struct ib_qp *qp) 448 { 449 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL; 450 } 451 452 static inline struct rxe_cq *to_rcq(struct ib_cq *cq) 453 { 454 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL; 455 } 456 457 static inline struct rxe_mem *to_rmr(struct ib_mr *mr) 458 { 459 return mr ? container_of(mr, struct rxe_mem, ibmr) : NULL; 460 } 461 462 static inline struct rxe_mem *to_rmw(struct ib_mw *mw) 463 { 464 return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL; 465 } 466 467 int rxe_register_device(struct rxe_dev *rxe); 468 int rxe_unregister_device(struct rxe_dev *rxe); 469 470 void rxe_mc_cleanup(struct rxe_pool_entry *arg); 471 472 #endif /* RXE_VERBS_H */ 473