1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #ifndef __MLX5_WQ_H__ 34 #define __MLX5_WQ_H__ 35 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/cq.h> 38 #include <linux/mlx5/qp.h> 39 40 struct mlx5_wq_param { 41 int buf_numa_node; 42 int db_numa_node; 43 }; 44 45 struct mlx5_wq_ctrl { 46 struct mlx5_core_dev *mdev; 47 struct mlx5_frag_buf buf; 48 struct mlx5_db db; 49 }; 50 51 struct mlx5_wq_cyc { 52 struct mlx5_frag_buf_ctrl fbc; 53 __be32 *db; 54 u16 sz; 55 u16 wqe_ctr; 56 u16 cur_sz; 57 }; 58 59 struct mlx5_wq_qp { 60 struct mlx5_wq_cyc rq; 61 struct mlx5_wq_cyc sq; 62 }; 63 64 struct mlx5_cqwq { 65 struct mlx5_frag_buf_ctrl fbc; 66 __be32 *db; 67 u32 cc; /* consumer counter */ 68 }; 69 70 struct mlx5_wq_ll { 71 struct mlx5_frag_buf_ctrl fbc; 72 __be32 *db; 73 __be16 *tail_next; 74 u16 head; 75 u16 wqe_ctr; 76 u16 cur_sz; 77 }; 78 79 int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 80 void *wqc, struct mlx5_wq_cyc *wq, 81 struct mlx5_wq_ctrl *wq_ctrl); 82 u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq); 83 84 int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 85 void *qpc, struct mlx5_wq_qp *wq, 86 struct mlx5_wq_ctrl *wq_ctrl); 87 88 int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 89 void *cqc, struct mlx5_cqwq *wq, 90 struct mlx5_wq_ctrl *wq_ctrl); 91 u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq); 92 93 int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 94 void *wqc, struct mlx5_wq_ll *wq, 95 struct mlx5_wq_ctrl *wq_ctrl); 96 u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq); 97 98 void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl); 99 100 static inline int mlx5_wq_cyc_is_full(struct mlx5_wq_cyc *wq) 101 { 102 return wq->cur_sz == wq->sz; 103 } 104 105 static inline int mlx5_wq_cyc_missing(struct mlx5_wq_cyc *wq) 106 { 107 return wq->sz - wq->cur_sz; 108 } 109 110 static inline int mlx5_wq_cyc_is_empty(struct mlx5_wq_cyc *wq) 111 { 112 return !wq->cur_sz; 113 } 114 115 static inline void mlx5_wq_cyc_push(struct mlx5_wq_cyc *wq) 116 { 117 wq->wqe_ctr++; 118 wq->cur_sz++; 119 } 120 121 static inline void mlx5_wq_cyc_push_n(struct mlx5_wq_cyc *wq, u8 n) 122 { 123 wq->wqe_ctr += n; 124 wq->cur_sz += n; 125 } 126 127 static inline void mlx5_wq_cyc_pop(struct mlx5_wq_cyc *wq) 128 { 129 wq->cur_sz--; 130 } 131 132 static inline void mlx5_wq_cyc_update_db_record(struct mlx5_wq_cyc *wq) 133 { 134 *wq->db = cpu_to_be32(wq->wqe_ctr); 135 } 136 137 static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr) 138 { 139 return ctr & wq->fbc.sz_m1; 140 } 141 142 static inline u16 mlx5_wq_cyc_get_head(struct mlx5_wq_cyc *wq) 143 { 144 return mlx5_wq_cyc_ctr2ix(wq, wq->wqe_ctr); 145 } 146 147 static inline u16 mlx5_wq_cyc_get_tail(struct mlx5_wq_cyc *wq) 148 { 149 return mlx5_wq_cyc_ctr2ix(wq, wq->wqe_ctr - wq->cur_sz); 150 } 151 152 static inline void *mlx5_wq_cyc_get_wqe(struct mlx5_wq_cyc *wq, u16 ix) 153 { 154 return mlx5_frag_buf_get_wqe(&wq->fbc, ix); 155 } 156 157 static inline u16 mlx5_wq_cyc_get_contig_wqebbs(struct mlx5_wq_cyc *wq, u16 ix) 158 { 159 return mlx5_frag_buf_get_idx_last_contig_stride(&wq->fbc, ix) - ix + 1; 160 } 161 162 static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2) 163 { 164 int equal = (cc1 == cc2); 165 int smaller = 0x8000 & (cc1 - cc2); 166 167 return !equal && !smaller; 168 } 169 170 static inline u32 mlx5_cqwq_ctr2ix(struct mlx5_cqwq *wq, u32 ctr) 171 { 172 return ctr & wq->fbc.sz_m1; 173 } 174 175 static inline u32 mlx5_cqwq_get_ci(struct mlx5_cqwq *wq) 176 { 177 return mlx5_cqwq_ctr2ix(wq, wq->cc); 178 } 179 180 static inline void *mlx5_cqwq_get_wqe(struct mlx5_cqwq *wq, u32 ix) 181 { 182 return mlx5_frag_buf_get_wqe(&wq->fbc, ix); 183 } 184 185 static inline u32 mlx5_cqwq_get_ctr_wrap_cnt(struct mlx5_cqwq *wq, u32 ctr) 186 { 187 return ctr >> wq->fbc.log_sz; 188 } 189 190 static inline u32 mlx5_cqwq_get_wrap_cnt(struct mlx5_cqwq *wq) 191 { 192 return mlx5_cqwq_get_ctr_wrap_cnt(wq, wq->cc); 193 } 194 195 static inline void mlx5_cqwq_pop(struct mlx5_cqwq *wq) 196 { 197 wq->cc++; 198 } 199 200 static inline void mlx5_cqwq_update_db_record(struct mlx5_cqwq *wq) 201 { 202 *wq->db = cpu_to_be32(wq->cc & 0xffffff); 203 } 204 205 static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq) 206 { 207 u32 ci = mlx5_cqwq_get_ci(wq); 208 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci); 209 u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK; 210 u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1; 211 212 if (cqe_ownership_bit != sw_ownership_val) 213 return NULL; 214 215 /* ensure cqe content is read after cqe ownership bit */ 216 dma_rmb(); 217 218 return cqe; 219 } 220 221 static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq) 222 { 223 return wq->cur_sz == wq->fbc.sz_m1; 224 } 225 226 static inline int mlx5_wq_ll_is_empty(struct mlx5_wq_ll *wq) 227 { 228 return !wq->cur_sz; 229 } 230 231 static inline int mlx5_wq_ll_missing(struct mlx5_wq_ll *wq) 232 { 233 return wq->fbc.sz_m1 - wq->cur_sz; 234 } 235 236 static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix) 237 { 238 return mlx5_frag_buf_get_wqe(&wq->fbc, ix); 239 } 240 241 static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next) 242 { 243 wq->head = head_next; 244 wq->wqe_ctr++; 245 wq->cur_sz++; 246 } 247 248 static inline void mlx5_wq_ll_pop(struct mlx5_wq_ll *wq, __be16 ix, 249 __be16 *next_tail_next) 250 { 251 *wq->tail_next = ix; 252 wq->tail_next = next_tail_next; 253 wq->cur_sz--; 254 } 255 256 static inline void mlx5_wq_ll_update_db_record(struct mlx5_wq_ll *wq) 257 { 258 *wq->db = cpu_to_be32(wq->wqe_ctr); 259 } 260 261 #endif /* __MLX5_WQ_H__ */ 262