1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #ifndef __QEDR_H__ 33 #define __QEDR_H__ 34 35 #include <linux/pci.h> 36 #include <linux/xarray.h> 37 #include <rdma/ib_addr.h> 38 #include <linux/qed/qed_if.h> 39 #include <linux/qed/qed_chain.h> 40 #include <linux/qed/qed_rdma_if.h> 41 #include <linux/qed/qede_rdma.h> 42 #include <linux/qed/roce_common.h> 43 #include <linux/completion.h> 44 #include "qedr_hsi_rdma.h" 45 46 #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA" 47 #define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev) 48 #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP) 49 #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE) 50 51 #define DP_DEBUG(dev, module, fmt, ...) \ 52 pr_debug("(%s) " module ": " fmt, \ 53 DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__) 54 55 #define QEDR_MSG_INIT "INIT" 56 #define QEDR_MSG_MISC "MISC" 57 #define QEDR_MSG_CQ " CQ" 58 #define QEDR_MSG_MR " MR" 59 #define QEDR_MSG_RQ " RQ" 60 #define QEDR_MSG_SQ " SQ" 61 #define QEDR_MSG_QP " QP" 62 #define QEDR_MSG_SRQ " SRQ" 63 #define QEDR_MSG_GSI " GSI" 64 #define QEDR_MSG_IWARP " IW" 65 66 #define QEDR_CQ_MAGIC_NUMBER (0x11223344) 67 68 #define FW_PAGE_SIZE (RDMA_RING_PAGE_SIZE) 69 #define FW_PAGE_SHIFT (12) 70 71 struct qedr_dev; 72 73 struct qedr_cnq { 74 struct qedr_dev *dev; 75 struct qed_chain pbl; 76 struct qed_sb_info *sb; 77 char name[32]; 78 u64 n_comp; 79 __le16 *hw_cons_ptr; 80 u8 index; 81 }; 82 83 #define QEDR_MAX_SGID 128 84 85 struct qedr_device_attr { 86 u32 vendor_id; 87 u32 vendor_part_id; 88 u32 hw_ver; 89 u64 fw_ver; 90 u64 node_guid; 91 u64 sys_image_guid; 92 u8 max_cnq; 93 u8 max_sge; 94 u16 max_inline; 95 u32 max_sqe; 96 u32 max_rqe; 97 u8 max_qp_resp_rd_atomic_resc; 98 u8 max_qp_req_rd_atomic_resc; 99 u64 max_dev_resp_rd_atomic_resc; 100 u32 max_cq; 101 u32 max_qp; 102 u32 max_mr; 103 u64 max_mr_size; 104 u32 max_cqe; 105 u32 max_mw; 106 u32 max_fmr; 107 u32 max_mr_mw_fmr_pbl; 108 u64 max_mr_mw_fmr_size; 109 u32 max_pd; 110 u32 max_ah; 111 u8 max_pkey; 112 u32 max_srq; 113 u32 max_srq_wr; 114 u8 max_srq_sge; 115 u8 max_stats_queues; 116 u32 dev_caps; 117 118 u64 page_size_caps; 119 u8 dev_ack_delay; 120 u32 reserved_lkey; 121 u32 bad_pkey_counter; 122 struct qed_rdma_events events; 123 }; 124 125 #define QEDR_ENET_STATE_BIT (0) 126 127 struct qedr_dev { 128 struct ib_device ibdev; 129 struct qed_dev *cdev; 130 struct pci_dev *pdev; 131 struct net_device *ndev; 132 133 enum ib_atomic_cap atomic_cap; 134 135 void *rdma_ctx; 136 struct qedr_device_attr attr; 137 138 const struct qed_rdma_ops *ops; 139 struct qed_int_info int_info; 140 141 struct qed_sb_info *sb_array; 142 struct qedr_cnq *cnq_array; 143 int num_cnq; 144 int sb_start; 145 146 void __iomem *db_addr; 147 u64 db_phys_addr; 148 u32 db_size; 149 u16 dpi; 150 151 union ib_gid *sgid_tbl; 152 153 /* Lock for sgid table */ 154 spinlock_t sgid_lock; 155 156 u64 guid; 157 158 u32 dp_module; 159 u8 dp_level; 160 u8 num_hwfns; 161 #define QEDR_IS_CMT(dev) ((dev)->num_hwfns > 1) 162 u8 affin_hwfn_idx; 163 u8 gsi_ll2_handle; 164 165 uint wq_multiplier; 166 u8 gsi_ll2_mac_address[ETH_ALEN]; 167 int gsi_qp_created; 168 struct qedr_cq *gsi_sqcq; 169 struct qedr_cq *gsi_rqcq; 170 struct qedr_qp *gsi_qp; 171 enum qed_rdma_type rdma_type; 172 struct xarray qps; 173 struct xarray srqs; 174 struct workqueue_struct *iwarp_wq; 175 u16 iwarp_max_mtu; 176 177 unsigned long enet_state; 178 179 u8 user_dpm_enabled; 180 }; 181 182 #define QEDR_MAX_SQ_PBL (0x8000) 183 #define QEDR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *)) 184 #define QEDR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge)) 185 #define QEDR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \ 186 QEDR_SQE_ELEMENT_SIZE) 187 #define QEDR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \ 188 QEDR_SQE_ELEMENT_SIZE) 189 #define QEDR_MAX_SQE ((QEDR_MAX_SQ_PBL_ENTRIES) *\ 190 (RDMA_RING_PAGE_SIZE) / \ 191 (QEDR_SQE_ELEMENT_SIZE) /\ 192 (QEDR_MAX_SQE_ELEMENTS_PER_SQE)) 193 /* RQ */ 194 #define QEDR_MAX_RQ_PBL (0x2000) 195 #define QEDR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *)) 196 #define QEDR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge)) 197 #define QEDR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE) 198 #define QEDR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \ 199 QEDR_RQE_ELEMENT_SIZE) 200 #define QEDR_MAX_RQE ((QEDR_MAX_RQ_PBL_ENTRIES) *\ 201 (RDMA_RING_PAGE_SIZE) / \ 202 (QEDR_RQE_ELEMENT_SIZE) /\ 203 (QEDR_MAX_RQE_ELEMENTS_PER_RQE)) 204 205 #define QEDR_CQE_SIZE (sizeof(union rdma_cqe)) 206 #define QEDR_MAX_CQE_PBL_SIZE (512 * 1024) 207 #define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \ 208 sizeof(u64)) - 1) 209 #define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \ 210 (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE)) 211 212 #define QEDR_ROCE_MAX_CNQ_SIZE (0x4000) 213 214 #define QEDR_MAX_PORT (1) 215 #define QEDR_PORT (1) 216 217 #define QEDR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME) 218 219 #define QEDR_ROCE_PKEY_MAX 1 220 #define QEDR_ROCE_PKEY_TABLE_LEN 1 221 #define QEDR_ROCE_PKEY_DEFAULT 0xffff 222 223 struct qedr_pbl { 224 struct list_head list_entry; 225 void *va; 226 dma_addr_t pa; 227 }; 228 229 struct qedr_ucontext { 230 struct ib_ucontext ibucontext; 231 struct qedr_dev *dev; 232 struct qedr_pd *pd; 233 void __iomem *dpi_addr; 234 struct rdma_user_mmap_entry *db_mmap_entry; 235 u64 dpi_phys_addr; 236 u32 dpi_size; 237 u16 dpi; 238 bool db_rec; 239 }; 240 241 union db_prod32 { 242 struct rdma_pwm_val16_data data; 243 u32 raw; 244 }; 245 246 union db_prod64 { 247 struct rdma_pwm_val32_data data; 248 u64 raw; 249 }; 250 251 enum qedr_cq_type { 252 QEDR_CQ_TYPE_GSI, 253 QEDR_CQ_TYPE_KERNEL, 254 QEDR_CQ_TYPE_USER, 255 }; 256 257 struct qedr_pbl_info { 258 u32 num_pbls; 259 u32 num_pbes; 260 u32 pbl_size; 261 u32 pbe_size; 262 bool two_layered; 263 }; 264 265 struct qedr_userq { 266 struct ib_umem *umem; 267 struct qedr_pbl_info pbl_info; 268 struct qedr_pbl *pbl_tbl; 269 u64 buf_addr; 270 size_t buf_len; 271 272 /* doorbell recovery */ 273 void __iomem *db_addr; 274 struct qedr_user_db_rec *db_rec_data; 275 struct rdma_user_mmap_entry *db_mmap_entry; 276 void __iomem *db_rec_db2_addr; 277 union db_prod32 db_rec_db2_data; 278 }; 279 280 struct qedr_cq { 281 struct ib_cq ibcq; 282 283 enum qedr_cq_type cq_type; 284 u32 sig; 285 286 u16 icid; 287 288 /* Lock to protect multiplem CQ's */ 289 spinlock_t cq_lock; 290 u8 arm_flags; 291 struct qed_chain pbl; 292 293 void __iomem *db_addr; 294 union db_prod64 db; 295 296 u8 pbl_toggle; 297 union rdma_cqe *latest_cqe; 298 union rdma_cqe *toggle_cqe; 299 300 u32 cq_cons; 301 302 struct qedr_userq q; 303 u8 destroyed; 304 u16 cnq_notif; 305 }; 306 307 struct qedr_pd { 308 struct ib_pd ibpd; 309 u32 pd_id; 310 struct qedr_ucontext *uctx; 311 }; 312 313 struct qedr_qp_hwq_info { 314 /* WQE Elements */ 315 struct qed_chain pbl; 316 u64 p_phys_addr_tbl; 317 u32 max_sges; 318 319 /* WQE */ 320 u16 prod; 321 u16 cons; 322 u16 wqe_cons; 323 u16 gsi_cons; 324 u16 max_wr; 325 326 /* DB */ 327 void __iomem *db; 328 union db_prod32 db_data; 329 330 void __iomem *iwarp_db2; 331 union db_prod32 iwarp_db2_data; 332 }; 333 334 #define QEDR_INC_SW_IDX(p_info, index) \ 335 do { \ 336 p_info->index = (p_info->index + 1) & \ 337 qed_chain_get_capacity(p_info->pbl) \ 338 } while (0) 339 340 struct qedr_srq_hwq_info { 341 u32 max_sges; 342 u32 max_wr; 343 struct qed_chain pbl; 344 u64 p_phys_addr_tbl; 345 u32 wqe_prod; 346 u32 sge_prod; 347 u32 wr_prod_cnt; 348 u32 wr_cons_cnt; 349 u32 num_elems; 350 351 u32 *virt_prod_pair_addr; 352 dma_addr_t phy_prod_pair_addr; 353 }; 354 355 struct qedr_srq { 356 struct ib_srq ibsrq; 357 struct qedr_dev *dev; 358 359 struct qedr_userq usrq; 360 struct qedr_srq_hwq_info hw_srq; 361 struct ib_umem *prod_umem; 362 u16 srq_id; 363 u32 srq_limit; 364 /* lock to protect srq recv post */ 365 spinlock_t lock; 366 }; 367 368 enum qedr_qp_err_bitmap { 369 QEDR_QP_ERR_SQ_FULL = 1, 370 QEDR_QP_ERR_RQ_FULL = 2, 371 QEDR_QP_ERR_BAD_SR = 4, 372 QEDR_QP_ERR_BAD_RR = 8, 373 QEDR_QP_ERR_SQ_PBL_FULL = 16, 374 QEDR_QP_ERR_RQ_PBL_FULL = 32, 375 }; 376 377 enum qedr_qp_create_type { 378 QEDR_QP_CREATE_NONE, 379 QEDR_QP_CREATE_USER, 380 QEDR_QP_CREATE_KERNEL, 381 }; 382 383 enum qedr_iwarp_cm_flags { 384 QEDR_IWARP_CM_WAIT_FOR_CONNECT = BIT(0), 385 QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1), 386 }; 387 388 struct qedr_qp { 389 struct ib_qp ibqp; /* must be first */ 390 struct qedr_dev *dev; 391 struct qedr_qp_hwq_info sq; 392 struct qedr_qp_hwq_info rq; 393 394 u32 max_inline_data; 395 396 /* Lock for QP's */ 397 spinlock_t q_lock; 398 struct qedr_cq *sq_cq; 399 struct qedr_cq *rq_cq; 400 struct qedr_srq *srq; 401 enum qed_roce_qp_state state; 402 u32 id; 403 struct qedr_pd *pd; 404 enum ib_qp_type qp_type; 405 enum qedr_qp_create_type create_type; 406 struct qed_rdma_qp *qed_qp; 407 u32 qp_id; 408 u16 icid; 409 u16 mtu; 410 int sgid_idx; 411 u32 rq_psn; 412 u32 sq_psn; 413 u32 qkey; 414 u32 dest_qp_num; 415 416 /* Relevant to qps created from kernel space only (ULPs) */ 417 u8 prev_wqe_size; 418 u16 wqe_cons; 419 u32 err_bitmap; 420 bool signaled; 421 422 /* SQ shadow */ 423 struct { 424 u64 wr_id; 425 enum ib_wc_opcode opcode; 426 u32 bytes_len; 427 u8 wqe_size; 428 bool signaled; 429 dma_addr_t icrc_mapping; 430 u32 *icrc; 431 struct qedr_mr *mr; 432 } *wqe_wr_id; 433 434 /* RQ shadow */ 435 struct { 436 u64 wr_id; 437 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE]; 438 u8 wqe_size; 439 440 u8 smac[ETH_ALEN]; 441 u16 vlan; 442 int rc; 443 } *rqe_wr_id; 444 445 /* Relevant to qps created from user space only (applications) */ 446 struct qedr_userq usq; 447 struct qedr_userq urq; 448 449 /* synchronization objects used with iwarp ep */ 450 struct kref refcnt; 451 struct completion iwarp_cm_comp; 452 unsigned long iwarp_cm_flags; /* enum iwarp_cm_flags */ 453 }; 454 455 struct qedr_ah { 456 struct ib_ah ibah; 457 struct rdma_ah_attr attr; 458 }; 459 460 enum qedr_mr_type { 461 QEDR_MR_USER, 462 QEDR_MR_KERNEL, 463 QEDR_MR_DMA, 464 QEDR_MR_FRMR, 465 }; 466 467 struct mr_info { 468 struct qedr_pbl *pbl_table; 469 struct qedr_pbl_info pbl_info; 470 struct list_head free_pbl_list; 471 struct list_head inuse_pbl_list; 472 u32 completed; 473 u32 completed_handled; 474 }; 475 476 struct qedr_mr { 477 struct ib_mr ibmr; 478 struct ib_umem *umem; 479 480 struct qed_rdma_register_tid_in_params hw_mr; 481 enum qedr_mr_type type; 482 483 struct qedr_dev *dev; 484 struct mr_info info; 485 486 u64 *pages; 487 u32 npages; 488 }; 489 490 struct qedr_user_mmap_entry { 491 struct rdma_user_mmap_entry rdma_entry; 492 struct qedr_dev *dev; 493 union { 494 u64 io_address; 495 void *address; 496 }; 497 size_t length; 498 u16 dpi; 499 u8 mmap_flag; 500 }; 501 502 #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT))) 503 504 #define QEDR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \ 505 RDMA_CQE_RESPONDER_IMM_FLG_SHIFT) 506 #define QEDR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \ 507 RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT) 508 #define QEDR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \ 509 RDMA_CQE_RESPONDER_INV_FLG_SHIFT) 510 511 static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info) 512 { 513 info->cons = (info->cons + 1) % info->max_wr; 514 info->wqe_cons++; 515 } 516 517 static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info) 518 { 519 info->prod = (info->prod + 1) % info->max_wr; 520 } 521 522 static inline int qedr_get_dmac(struct qedr_dev *dev, 523 struct rdma_ah_attr *ah_attr, u8 *mac_addr) 524 { 525 union ib_gid zero_sgid = { { 0 } }; 526 struct in6_addr in6; 527 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 528 u8 *dmac; 529 530 if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) { 531 DP_ERR(dev, "Local port GID not supported\n"); 532 eth_zero_addr(mac_addr); 533 return -EINVAL; 534 } 535 536 memcpy(&in6, grh->dgid.raw, sizeof(in6)); 537 dmac = rdma_ah_retrieve_dmac(ah_attr); 538 if (!dmac) 539 return -EINVAL; 540 ether_addr_copy(mac_addr, dmac); 541 542 return 0; 543 } 544 545 struct qedr_iw_listener { 546 struct qedr_dev *dev; 547 struct iw_cm_id *cm_id; 548 int backlog; 549 void *qed_handle; 550 }; 551 552 struct qedr_iw_ep { 553 struct qedr_dev *dev; 554 struct iw_cm_id *cm_id; 555 struct qedr_qp *qp; 556 void *qed_context; 557 struct kref refcnt; 558 }; 559 560 static inline 561 struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext) 562 { 563 return container_of(ibucontext, struct qedr_ucontext, ibucontext); 564 } 565 566 static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev) 567 { 568 return container_of(ibdev, struct qedr_dev, ibdev); 569 } 570 571 static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd) 572 { 573 return container_of(ibpd, struct qedr_pd, ibpd); 574 } 575 576 static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq) 577 { 578 return container_of(ibcq, struct qedr_cq, ibcq); 579 } 580 581 static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp) 582 { 583 return container_of(ibqp, struct qedr_qp, ibqp); 584 } 585 586 static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah) 587 { 588 return container_of(ibah, struct qedr_ah, ibah); 589 } 590 591 static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr) 592 { 593 return container_of(ibmr, struct qedr_mr, ibmr); 594 } 595 596 static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq) 597 { 598 return container_of(ibsrq, struct qedr_srq, ibsrq); 599 } 600 601 static inline struct qedr_user_mmap_entry * 602 get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry) 603 { 604 return container_of(rdma_entry, struct qedr_user_mmap_entry, 605 rdma_entry); 606 } 607 #endif 608