1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: Fast Path Operators (header) 37 */ 38 39 #ifndef __BNXT_QPLIB_FP_H__ 40 #define __BNXT_QPLIB_FP_H__ 41 42 struct bnxt_qplib_sge { 43 u64 addr; 44 u32 lkey; 45 u32 size; 46 }; 47 48 #define BNXT_QPLIB_MAX_SQE_ENTRY_SIZE sizeof(struct sq_send) 49 50 #define SQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_SQE_ENTRY_SIZE) 51 #define SQE_MAX_IDX_PER_PG (SQE_CNT_PER_PG - 1) 52 53 static inline u32 get_sqe_pg(u32 val) 54 { 55 return ((val & ~SQE_MAX_IDX_PER_PG) / SQE_CNT_PER_PG); 56 } 57 58 static inline u32 get_sqe_idx(u32 val) 59 { 60 return (val & SQE_MAX_IDX_PER_PG); 61 } 62 63 #define BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE sizeof(struct sq_psn_search) 64 65 #define PSNE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE) 66 #define PSNE_MAX_IDX_PER_PG (PSNE_CNT_PER_PG - 1) 67 68 static inline u32 get_psne_pg(u32 val) 69 { 70 return ((val & ~PSNE_MAX_IDX_PER_PG) / PSNE_CNT_PER_PG); 71 } 72 73 static inline u32 get_psne_idx(u32 val) 74 { 75 return (val & PSNE_MAX_IDX_PER_PG); 76 } 77 78 #define BNXT_QPLIB_QP_MAX_SGL 6 79 80 struct bnxt_qplib_swq { 81 u64 wr_id; 82 u8 type; 83 u8 flags; 84 u32 start_psn; 85 u32 next_psn; 86 struct sq_psn_search *psn_search; 87 }; 88 89 struct bnxt_qplib_swqe { 90 /* General */ 91 #define BNXT_QPLIB_FENCE_WRID 0x46454E43 /* "FENC" */ 92 u64 wr_id; 93 u8 reqs_type; 94 u8 type; 95 #define BNXT_QPLIB_SWQE_TYPE_SEND 0 96 #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 97 #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 98 #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 99 #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 100 #define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 101 #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 102 #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 103 #define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 104 #define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 105 #define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 106 #define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 107 #define BNXT_QPLIB_SWQE_TYPE_RECV 128 108 #define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 109 u8 flags; 110 #define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0) 111 #define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1) 112 #define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2) 113 #define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3) 114 #define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4) 115 struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL]; 116 int num_sge; 117 /* Max inline data is 96 bytes */ 118 u32 inline_len; 119 #define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96 120 u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH]; 121 122 union { 123 /* Send, with imm, inval key */ 124 struct { 125 union { 126 __be32 imm_data; 127 u32 inv_key; 128 }; 129 u32 q_key; 130 u32 dst_qp; 131 u16 avid; 132 } send; 133 134 /* Send Raw Ethernet and QP1 */ 135 struct { 136 u16 lflags; 137 u16 cfa_action; 138 u32 cfa_meta; 139 } rawqp1; 140 141 /* RDMA write, with imm, read */ 142 struct { 143 union { 144 __be32 imm_data; 145 u32 inv_key; 146 }; 147 u64 remote_va; 148 u32 r_key; 149 } rdma; 150 151 /* Atomic cmp/swap, fetch/add */ 152 struct { 153 u64 remote_va; 154 u32 r_key; 155 u64 swap_data; 156 u64 cmp_data; 157 } atomic; 158 159 /* Local Invalidate */ 160 struct { 161 u32 inv_l_key; 162 } local_inv; 163 164 /* FR-PMR */ 165 struct { 166 u8 access_cntl; 167 u8 pg_sz_log; 168 bool zero_based; 169 u32 l_key; 170 u32 length; 171 u8 pbl_pg_sz_log; 172 #define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 173 #define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 174 #define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 175 #define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 176 #define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 177 #define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 178 #define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 179 #define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 180 u8 levels; 181 #define PAGE_SHIFT_4K 12 182 __le64 *pbl_ptr; 183 dma_addr_t pbl_dma_ptr; 184 u64 *page_list; 185 u16 page_list_len; 186 u64 va; 187 } frmr; 188 189 /* Bind */ 190 struct { 191 u8 access_cntl; 192 #define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0) 193 #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1) 194 #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2) 195 #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3) 196 #define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4) 197 bool zero_based; 198 u8 mw_type; 199 u32 parent_l_key; 200 u32 r_key; 201 u64 va; 202 u32 length; 203 } bind; 204 }; 205 }; 206 207 #define BNXT_QPLIB_MAX_RQE_ENTRY_SIZE sizeof(struct rq_wqe) 208 209 #define RQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_RQE_ENTRY_SIZE) 210 #define RQE_MAX_IDX_PER_PG (RQE_CNT_PER_PG - 1) 211 #define RQE_PG(x) (((x) & ~RQE_MAX_IDX_PER_PG) / RQE_CNT_PER_PG) 212 #define RQE_IDX(x) ((x) & RQE_MAX_IDX_PER_PG) 213 214 struct bnxt_qplib_q { 215 struct bnxt_qplib_hwq hwq; 216 struct bnxt_qplib_swq *swq; 217 struct scatterlist *sglist; 218 u32 nmap; 219 u32 max_wqe; 220 u16 q_full_delta; 221 u16 max_sge; 222 u32 psn; 223 bool condition; 224 bool single; 225 bool send_phantom; 226 u32 phantom_wqe_cnt; 227 u32 phantom_cqe_cnt; 228 u32 next_cq_cons; 229 bool flushed; 230 }; 231 232 struct bnxt_qplib_qp { 233 struct bnxt_qplib_pd *pd; 234 struct bnxt_qplib_dpi *dpi; 235 u64 qp_handle; 236 #define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF 237 u32 id; 238 u8 type; 239 u8 sig_type; 240 u32 modify_flags; 241 u8 state; 242 u8 cur_qp_state; 243 u32 max_inline_data; 244 u32 mtu; 245 u8 path_mtu; 246 bool en_sqd_async_notify; 247 u16 pkey_index; 248 u32 qkey; 249 u32 dest_qp_id; 250 u8 access; 251 u8 timeout; 252 u8 retry_cnt; 253 u8 rnr_retry; 254 u64 wqe_cnt; 255 u32 min_rnr_timer; 256 u32 max_rd_atomic; 257 u32 max_dest_rd_atomic; 258 u32 dest_qpn; 259 u8 smac[6]; 260 u16 vlan_id; 261 u8 nw_type; 262 struct bnxt_qplib_ah ah; 263 264 #define BTH_PSN_MASK ((1 << 24) - 1) 265 /* SQ */ 266 struct bnxt_qplib_q sq; 267 /* RQ */ 268 struct bnxt_qplib_q rq; 269 /* SRQ */ 270 struct bnxt_qplib_srq *srq; 271 /* CQ */ 272 struct bnxt_qplib_cq *scq; 273 struct bnxt_qplib_cq *rcq; 274 /* IRRQ and ORRQ */ 275 struct bnxt_qplib_hwq irrq; 276 struct bnxt_qplib_hwq orrq; 277 /* Header buffer for QP1 */ 278 int sq_hdr_buf_size; 279 int rq_hdr_buf_size; 280 /* 281 * Buffer space for ETH(14), IP or GRH(40), UDP header(8) 282 * and ib_bth + ib_deth (20). 283 * Max required is 82 when RoCE V2 is enabled 284 */ 285 #define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 286 /* Ethernet header = 14 */ 287 /* ib_grh = 40 (provided by MAD) */ 288 /* ib_bth + ib_deth = 20 */ 289 /* MAD = 256 (provided by MAD) */ 290 /* iCRC = 4 */ 291 #define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 292 #define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 293 #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 294 #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 295 #define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 296 void *sq_hdr_buf; 297 dma_addr_t sq_hdr_buf_map; 298 void *rq_hdr_buf; 299 dma_addr_t rq_hdr_buf_map; 300 struct list_head sq_flush; 301 struct list_head rq_flush; 302 }; 303 304 #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) 305 306 #define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE) 307 #define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1) 308 #define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG) 309 #define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG) 310 311 #define ROCE_CQE_CMP_V 0 312 #define CQE_CMP_VALID(hdr, raw_cons, cp_bit) \ 313 (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ 314 !((raw_cons) & (cp_bit))) 315 316 static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *qplib_q) 317 { 318 return HWQ_CMP((qplib_q->hwq.prod + qplib_q->q_full_delta), 319 &qplib_q->hwq) == HWQ_CMP(qplib_q->hwq.cons, 320 &qplib_q->hwq); 321 } 322 323 struct bnxt_qplib_cqe { 324 u8 status; 325 u8 type; 326 u8 opcode; 327 u32 length; 328 u64 wr_id; 329 union { 330 __be32 immdata; 331 u32 invrkey; 332 }; 333 u64 qp_handle; 334 u64 mr_handle; 335 u16 flags; 336 u8 smac[6]; 337 u32 src_qp; 338 u16 raweth_qp1_flags; 339 u16 raweth_qp1_errors; 340 u16 raweth_qp1_cfa_code; 341 u32 raweth_qp1_flags2; 342 u32 raweth_qp1_metadata; 343 u8 raweth_qp1_payload_offset; 344 u16 pkey_index; 345 }; 346 347 #define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 348 struct bnxt_qplib_cq { 349 struct bnxt_qplib_dpi *dpi; 350 void __iomem *dbr_base; 351 u32 max_wqe; 352 u32 id; 353 u16 count; 354 u16 period; 355 struct bnxt_qplib_hwq hwq; 356 u32 cnq_hw_ring_id; 357 struct bnxt_qplib_nq *nq; 358 bool resize_in_progress; 359 struct scatterlist *sghead; 360 u32 nmap; 361 u64 cq_handle; 362 363 #define CQ_RESIZE_WAIT_TIME_MS 500 364 unsigned long flags; 365 #define CQ_FLAGS_RESIZE_IN_PROG 1 366 wait_queue_head_t waitq; 367 struct list_head sqf_head, rqf_head; 368 atomic_t arm_state; 369 spinlock_t compl_lock; /* synch CQ handlers */ 370 }; 371 372 #define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) 373 #define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) 374 #define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2) 375 #define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1) 376 #define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1) 377 #define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1) 378 379 #define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base) 380 381 #define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE) 382 #define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1) 383 #define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG) 384 #define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG) 385 386 #define NQE_CMP_VALID(hdr, raw_cons, cp_bit) \ 387 (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \ 388 !((raw_cons) & (cp_bit))) 389 390 #define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) 391 392 #define NQ_CONS_PCI_BAR_REGION 2 393 #define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) 394 #define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID 395 #define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK 396 #define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \ 397 NQ_DB_IDX_VALID) 398 #define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \ 399 NQ_DB_IDX_VALID | \ 400 NQ_DB_IRQ_DIS) 401 #define NQ_DB_REARM(db, raw_cons, cp_bit) \ 402 writel(NQ_DB_CP_FLAGS_REARM | ((raw_cons) & ((cp_bit) - 1)), db) 403 #define NQ_DB(db, raw_cons, cp_bit) \ 404 writel(NQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db) 405 406 struct bnxt_qplib_nq { 407 struct pci_dev *pdev; 408 409 int vector; 410 cpumask_t mask; 411 int budget; 412 bool requested; 413 struct tasklet_struct worker; 414 struct bnxt_qplib_hwq hwq; 415 416 u16 bar_reg; 417 u16 bar_reg_off; 418 u16 ring_id; 419 void __iomem *bar_reg_iomem; 420 421 int (*cqn_handler) 422 (struct bnxt_qplib_nq *nq, 423 struct bnxt_qplib_cq *cq); 424 int (*srqn_handler) 425 (struct bnxt_qplib_nq *nq, 426 void *srq, 427 u8 event); 428 struct workqueue_struct *cqn_wq; 429 char name[32]; 430 }; 431 432 struct bnxt_qplib_nq_work { 433 struct work_struct work; 434 struct bnxt_qplib_nq *nq; 435 struct bnxt_qplib_cq *cq; 436 }; 437 438 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); 439 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, 440 int nq_idx, int msix_vector, int bar_reg_offset, 441 int (*cqn_handler)(struct bnxt_qplib_nq *nq, 442 struct bnxt_qplib_cq *cq), 443 int (*srqn_handler)(struct bnxt_qplib_nq *nq, 444 void *srq, 445 u8 event)); 446 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); 447 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); 448 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); 449 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); 450 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); 451 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, 452 struct bnxt_qplib_sge *sge); 453 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, 454 struct bnxt_qplib_sge *sge); 455 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); 456 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, 457 u32 index); 458 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); 459 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, 460 struct bnxt_qplib_swqe *wqe); 461 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); 462 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, 463 struct bnxt_qplib_swqe *wqe); 464 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); 465 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); 466 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, 467 int num, struct bnxt_qplib_qp **qp); 468 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq); 469 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); 470 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq); 471 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq); 472 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp); 473 void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp); 474 void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp, 475 unsigned long *flags); 476 void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp, 477 unsigned long *flags); 478 int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, 479 struct bnxt_qplib_cqe *cqe, 480 int num_cqes); 481 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); 482 #endif /* __BNXT_QPLIB_FP_H__ */ 483