1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/kernel.h> 34 #include <linux/module.h> 35 #include <linux/hardirq.h> 36 #include <linux/mlx5/driver.h> 37 #include <linux/mlx5/cmd.h> 38 #include <rdma/ib_verbs.h> 39 #include <linux/mlx5/cq.h> 40 #include "mlx5_core.h" 41 42 #define TASKLET_MAX_TIME 2 43 #define TASKLET_MAX_TIME_JIFFIES msecs_to_jiffies(TASKLET_MAX_TIME) 44 45 void mlx5_cq_tasklet_cb(unsigned long data) 46 { 47 unsigned long flags; 48 unsigned long end = jiffies + TASKLET_MAX_TIME_JIFFIES; 49 struct mlx5_eq_tasklet *ctx = (struct mlx5_eq_tasklet *)data; 50 struct mlx5_core_cq *mcq; 51 struct mlx5_core_cq *temp; 52 53 spin_lock_irqsave(&ctx->lock, flags); 54 list_splice_tail_init(&ctx->list, &ctx->process_list); 55 spin_unlock_irqrestore(&ctx->lock, flags); 56 57 list_for_each_entry_safe(mcq, temp, &ctx->process_list, 58 tasklet_ctx.list) { 59 list_del_init(&mcq->tasklet_ctx.list); 60 mcq->tasklet_ctx.comp(mcq); 61 if (atomic_dec_and_test(&mcq->refcount)) 62 complete(&mcq->free); 63 if (time_after(jiffies, end)) 64 break; 65 } 66 67 if (!list_empty(&ctx->process_list)) 68 tasklet_schedule(&ctx->task); 69 } 70 71 static void mlx5_add_cq_to_tasklet(struct mlx5_core_cq *cq) 72 { 73 unsigned long flags; 74 struct mlx5_eq_tasklet *tasklet_ctx = cq->tasklet_ctx.priv; 75 76 spin_lock_irqsave(&tasklet_ctx->lock, flags); 77 /* When migrating CQs between EQs will be implemented, please note 78 * that you need to sync this point. It is possible that 79 * while migrating a CQ, completions on the old EQs could 80 * still arrive. 81 */ 82 if (list_empty_careful(&cq->tasklet_ctx.list)) { 83 atomic_inc(&cq->refcount); 84 list_add_tail(&cq->tasklet_ctx.list, &tasklet_ctx->list); 85 } 86 spin_unlock_irqrestore(&tasklet_ctx->lock, flags); 87 } 88 89 void mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn) 90 { 91 struct mlx5_core_cq *cq; 92 struct mlx5_cq_table *table = &dev->priv.cq_table; 93 94 spin_lock(&table->lock); 95 cq = radix_tree_lookup(&table->tree, cqn); 96 if (likely(cq)) 97 atomic_inc(&cq->refcount); 98 spin_unlock(&table->lock); 99 100 if (!cq) { 101 mlx5_core_warn(dev, "Completion event for bogus CQ 0x%x\n", cqn); 102 return; 103 } 104 105 ++cq->arm_sn; 106 107 cq->comp(cq); 108 109 if (atomic_dec_and_test(&cq->refcount)) 110 complete(&cq->free); 111 } 112 113 void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type) 114 { 115 struct mlx5_cq_table *table = &dev->priv.cq_table; 116 struct mlx5_core_cq *cq; 117 118 spin_lock(&table->lock); 119 120 cq = radix_tree_lookup(&table->tree, cqn); 121 if (cq) 122 atomic_inc(&cq->refcount); 123 124 spin_unlock(&table->lock); 125 126 if (!cq) { 127 mlx5_core_warn(dev, "Async event for bogus CQ 0x%x\n", cqn); 128 return; 129 } 130 131 cq->event(cq, event_type); 132 133 if (atomic_dec_and_test(&cq->refcount)) 134 complete(&cq->free); 135 } 136 137 138 int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 139 struct mlx5_create_cq_mbox_in *in, int inlen) 140 { 141 int err; 142 struct mlx5_cq_table *table = &dev->priv.cq_table; 143 struct mlx5_create_cq_mbox_out out; 144 struct mlx5_destroy_cq_mbox_in din; 145 struct mlx5_destroy_cq_mbox_out dout; 146 int eqn = MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context), 147 c_eqn); 148 struct mlx5_eq *eq; 149 150 eq = mlx5_eqn2eq(dev, eqn); 151 if (IS_ERR(eq)) 152 return PTR_ERR(eq); 153 154 in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_CQ); 155 memset(&out, 0, sizeof(out)); 156 err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); 157 if (err) 158 return err; 159 160 if (out.hdr.status) 161 return mlx5_cmd_status_to_err(&out.hdr); 162 163 cq->cqn = be32_to_cpu(out.cqn) & 0xffffff; 164 cq->cons_index = 0; 165 cq->arm_sn = 0; 166 atomic_set(&cq->refcount, 1); 167 init_completion(&cq->free); 168 if (!cq->comp) 169 cq->comp = mlx5_add_cq_to_tasklet; 170 /* assuming CQ will be deleted before the EQ */ 171 cq->tasklet_ctx.priv = &eq->tasklet_ctx; 172 INIT_LIST_HEAD(&cq->tasklet_ctx.list); 173 174 spin_lock_irq(&table->lock); 175 err = radix_tree_insert(&table->tree, cq->cqn, cq); 176 spin_unlock_irq(&table->lock); 177 if (err) 178 goto err_cmd; 179 180 cq->pid = current->pid; 181 err = mlx5_debug_cq_add(dev, cq); 182 if (err) 183 mlx5_core_dbg(dev, "failed adding CP 0x%x to debug file system\n", 184 cq->cqn); 185 186 return 0; 187 188 err_cmd: 189 memset(&din, 0, sizeof(din)); 190 memset(&dout, 0, sizeof(dout)); 191 din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ); 192 mlx5_cmd_exec(dev, &din, sizeof(din), &dout, sizeof(dout)); 193 return err; 194 } 195 EXPORT_SYMBOL(mlx5_core_create_cq); 196 197 int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq) 198 { 199 struct mlx5_cq_table *table = &dev->priv.cq_table; 200 struct mlx5_destroy_cq_mbox_in in; 201 struct mlx5_destroy_cq_mbox_out out; 202 struct mlx5_core_cq *tmp; 203 int err; 204 205 spin_lock_irq(&table->lock); 206 tmp = radix_tree_delete(&table->tree, cq->cqn); 207 spin_unlock_irq(&table->lock); 208 if (!tmp) { 209 mlx5_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn); 210 return -EINVAL; 211 } 212 if (tmp != cq) { 213 mlx5_core_warn(dev, "corruption on srqn 0x%x\n", cq->cqn); 214 return -EINVAL; 215 } 216 217 memset(&in, 0, sizeof(in)); 218 memset(&out, 0, sizeof(out)); 219 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ); 220 in.cqn = cpu_to_be32(cq->cqn); 221 err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 222 if (err) 223 return err; 224 225 if (out.hdr.status) 226 return mlx5_cmd_status_to_err(&out.hdr); 227 228 synchronize_irq(cq->irqn); 229 230 mlx5_debug_cq_remove(dev, cq); 231 if (atomic_dec_and_test(&cq->refcount)) 232 complete(&cq->free); 233 wait_for_completion(&cq->free); 234 235 return 0; 236 } 237 EXPORT_SYMBOL(mlx5_core_destroy_cq); 238 239 int mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 240 struct mlx5_query_cq_mbox_out *out) 241 { 242 struct mlx5_query_cq_mbox_in in; 243 int err; 244 245 memset(&in, 0, sizeof(in)); 246 memset(out, 0, sizeof(*out)); 247 248 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_CQ); 249 in.cqn = cpu_to_be32(cq->cqn); 250 err = mlx5_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); 251 if (err) 252 return err; 253 254 if (out->hdr.status) 255 return mlx5_cmd_status_to_err(&out->hdr); 256 257 return err; 258 } 259 EXPORT_SYMBOL(mlx5_core_query_cq); 260 261 262 int mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 263 struct mlx5_modify_cq_mbox_in *in, int in_sz) 264 { 265 struct mlx5_modify_cq_mbox_out out; 266 int err; 267 268 memset(&out, 0, sizeof(out)); 269 in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MODIFY_CQ); 270 err = mlx5_cmd_exec(dev, in, in_sz, &out, sizeof(out)); 271 if (err) 272 return err; 273 274 if (out.hdr.status) 275 return mlx5_cmd_status_to_err(&out.hdr); 276 277 return 0; 278 } 279 EXPORT_SYMBOL(mlx5_core_modify_cq); 280 281 int mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev, 282 struct mlx5_core_cq *cq, 283 u16 cq_period, 284 u16 cq_max_count) 285 { 286 struct mlx5_modify_cq_mbox_in in; 287 288 memset(&in, 0, sizeof(in)); 289 290 in.cqn = cpu_to_be32(cq->cqn); 291 in.ctx.cq_period = cpu_to_be16(cq_period); 292 in.ctx.cq_max_count = cpu_to_be16(cq_max_count); 293 in.field_select = cpu_to_be32(MLX5_CQ_MODIFY_PERIOD | 294 MLX5_CQ_MODIFY_COUNT); 295 296 return mlx5_core_modify_cq(dev, cq, &in, sizeof(in)); 297 } 298 299 int mlx5_init_cq_table(struct mlx5_core_dev *dev) 300 { 301 struct mlx5_cq_table *table = &dev->priv.cq_table; 302 int err; 303 304 memset(table, 0, sizeof(*table)); 305 spin_lock_init(&table->lock); 306 INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 307 err = mlx5_cq_debugfs_init(dev); 308 309 return err; 310 } 311 312 void mlx5_cleanup_cq_table(struct mlx5_core_dev *dev) 313 { 314 mlx5_cq_debugfs_cleanup(dev); 315 } 316