1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2021, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/interrupt.h> 7 #include <linux/notifier.h> 8 #include <linux/module.h> 9 #include <linux/mlx5/driver.h> 10 #include <linux/mlx5/vport.h> 11 #include <linux/mlx5/eq.h> 12 #ifdef CONFIG_RFS_ACCEL 13 #include <linux/cpu_rmap.h> 14 #endif 15 #include "mlx5_core.h" 16 #include "lib/eq.h" 17 #include "fpga/core.h" 18 #include "eswitch.h" 19 #include "lib/clock.h" 20 #include "diag/fw_tracer.h" 21 #include "mlx5_irq.h" 22 #include "devlink.h" 23 24 enum { 25 MLX5_EQE_OWNER_INIT_VAL = 0x1, 26 }; 27 28 enum { 29 MLX5_EQ_STATE_ARMED = 0x9, 30 MLX5_EQ_STATE_FIRED = 0xa, 31 MLX5_EQ_STATE_ALWAYS_ARMED = 0xb, 32 }; 33 34 enum { 35 MLX5_EQ_DOORBEL_OFFSET = 0x40, 36 }; 37 38 /* budget must be smaller than MLX5_NUM_SPARE_EQE to guarantee that we update 39 * the ci before we polled all the entries in the EQ. MLX5_NUM_SPARE_EQE is 40 * used to set the EQ size, budget must be smaller than the EQ size. 41 */ 42 enum { 43 MLX5_EQ_POLLING_BUDGET = 128, 44 }; 45 46 static_assert(MLX5_EQ_POLLING_BUDGET <= MLX5_NUM_SPARE_EQE); 47 48 struct mlx5_eq_table { 49 struct list_head comp_eqs_list; 50 struct mlx5_eq_async pages_eq; 51 struct mlx5_eq_async cmd_eq; 52 struct mlx5_eq_async async_eq; 53 54 struct atomic_notifier_head nh[MLX5_EVENT_TYPE_MAX]; 55 56 /* Since CQ DB is stored in async_eq */ 57 struct mlx5_nb cq_err_nb; 58 59 struct mutex lock; /* sync async eqs creations */ 60 int num_comp_eqs; 61 struct mlx5_irq_table *irq_table; 62 struct mlx5_irq **comp_irqs; 63 struct mlx5_irq *ctrl_irq; 64 #ifdef CONFIG_RFS_ACCEL 65 struct cpu_rmap *rmap; 66 #endif 67 }; 68 69 #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG) | \ 70 (1ull << MLX5_EVENT_TYPE_COMM_EST) | \ 71 (1ull << MLX5_EVENT_TYPE_SQ_DRAINED) | \ 72 (1ull << MLX5_EVENT_TYPE_CQ_ERROR) | \ 73 (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR) | \ 74 (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED) | \ 75 (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ 76 (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR) | \ 77 (1ull << MLX5_EVENT_TYPE_PORT_CHANGE) | \ 78 (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR) | \ 79 (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE) | \ 80 (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT)) 81 82 static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn) 83 { 84 u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {}; 85 86 MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); 87 MLX5_SET(destroy_eq_in, in, eq_number, eqn); 88 return mlx5_cmd_exec_in(dev, destroy_eq, in); 89 } 90 91 /* caller must eventually call mlx5_cq_put on the returned cq */ 92 static struct mlx5_core_cq *mlx5_eq_cq_get(struct mlx5_eq *eq, u32 cqn) 93 { 94 struct mlx5_cq_table *table = &eq->cq_table; 95 struct mlx5_core_cq *cq = NULL; 96 97 rcu_read_lock(); 98 cq = radix_tree_lookup(&table->tree, cqn); 99 if (likely(cq)) 100 mlx5_cq_hold(cq); 101 rcu_read_unlock(); 102 103 return cq; 104 } 105 106 static int mlx5_eq_comp_int(struct notifier_block *nb, 107 __always_unused unsigned long action, 108 __always_unused void *data) 109 { 110 struct mlx5_eq_comp *eq_comp = 111 container_of(nb, struct mlx5_eq_comp, irq_nb); 112 struct mlx5_eq *eq = &eq_comp->core; 113 struct mlx5_eqe *eqe; 114 int num_eqes = 0; 115 u32 cqn = -1; 116 117 eqe = next_eqe_sw(eq); 118 if (!eqe) 119 goto out; 120 121 do { 122 struct mlx5_core_cq *cq; 123 124 /* Make sure we read EQ entry contents after we've 125 * checked the ownership bit. 126 */ 127 dma_rmb(); 128 /* Assume (eqe->type) is always MLX5_EVENT_TYPE_COMP */ 129 cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff; 130 131 cq = mlx5_eq_cq_get(eq, cqn); 132 if (likely(cq)) { 133 ++cq->arm_sn; 134 cq->comp(cq, eqe); 135 mlx5_cq_put(cq); 136 } else { 137 dev_dbg_ratelimited(eq->dev->device, 138 "Completion event for bogus CQ 0x%x\n", cqn); 139 } 140 141 ++eq->cons_index; 142 143 } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq))); 144 145 out: 146 eq_update_ci(eq, 1); 147 148 if (cqn != -1) 149 tasklet_schedule(&eq_comp->tasklet_ctx.task); 150 151 return 0; 152 } 153 154 /* Some architectures don't latch interrupts when they are disabled, so using 155 * mlx5_eq_poll_irq_disabled could end up losing interrupts while trying to 156 * avoid losing them. It is not recommended to use it, unless this is the last 157 * resort. 158 */ 159 u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq) 160 { 161 u32 count_eqe; 162 163 disable_irq(eq->core.irqn); 164 count_eqe = eq->core.cons_index; 165 mlx5_eq_comp_int(&eq->irq_nb, 0, NULL); 166 count_eqe = eq->core.cons_index - count_eqe; 167 enable_irq(eq->core.irqn); 168 169 return count_eqe; 170 } 171 172 static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, bool recovery, 173 unsigned long *flags) 174 __acquires(&eq->lock) 175 { 176 if (!recovery) 177 spin_lock(&eq->lock); 178 else 179 spin_lock_irqsave(&eq->lock, *flags); 180 } 181 182 static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, bool recovery, 183 unsigned long *flags) 184 __releases(&eq->lock) 185 { 186 if (!recovery) 187 spin_unlock(&eq->lock); 188 else 189 spin_unlock_irqrestore(&eq->lock, *flags); 190 } 191 192 enum async_eq_nb_action { 193 ASYNC_EQ_IRQ_HANDLER = 0, 194 ASYNC_EQ_RECOVER = 1, 195 }; 196 197 static int mlx5_eq_async_int(struct notifier_block *nb, 198 unsigned long action, void *data) 199 { 200 struct mlx5_eq_async *eq_async = 201 container_of(nb, struct mlx5_eq_async, irq_nb); 202 struct mlx5_eq *eq = &eq_async->core; 203 struct mlx5_eq_table *eqt; 204 struct mlx5_core_dev *dev; 205 struct mlx5_eqe *eqe; 206 unsigned long flags; 207 int num_eqes = 0; 208 bool recovery; 209 210 dev = eq->dev; 211 eqt = dev->priv.eq_table; 212 213 recovery = action == ASYNC_EQ_RECOVER; 214 mlx5_eq_async_int_lock(eq_async, recovery, &flags); 215 216 eqe = next_eqe_sw(eq); 217 if (!eqe) 218 goto out; 219 220 do { 221 /* 222 * Make sure we read EQ entry contents after we've 223 * checked the ownership bit. 224 */ 225 dma_rmb(); 226 227 atomic_notifier_call_chain(&eqt->nh[eqe->type], eqe->type, eqe); 228 atomic_notifier_call_chain(&eqt->nh[MLX5_EVENT_TYPE_NOTIFY_ANY], eqe->type, eqe); 229 230 ++eq->cons_index; 231 232 } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq))); 233 234 out: 235 eq_update_ci(eq, 1); 236 mlx5_eq_async_int_unlock(eq_async, recovery, &flags); 237 238 return unlikely(recovery) ? num_eqes : 0; 239 } 240 241 void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev) 242 { 243 struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq; 244 int eqes; 245 246 eqes = mlx5_eq_async_int(&eq->irq_nb, ASYNC_EQ_RECOVER, NULL); 247 if (eqes) 248 mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes); 249 } 250 251 static void init_eq_buf(struct mlx5_eq *eq) 252 { 253 struct mlx5_eqe *eqe; 254 int i; 255 256 for (i = 0; i < eq_get_size(eq); i++) { 257 eqe = get_eqe(eq, i); 258 eqe->owner = MLX5_EQE_OWNER_INIT_VAL; 259 } 260 } 261 262 static int 263 create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, 264 struct mlx5_eq_param *param) 265 { 266 u8 log_eq_size = order_base_2(param->nent + MLX5_NUM_SPARE_EQE); 267 struct mlx5_cq_table *cq_table = &eq->cq_table; 268 u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0}; 269 u8 log_eq_stride = ilog2(MLX5_EQE_SIZE); 270 struct mlx5_priv *priv = &dev->priv; 271 __be64 *pas; 272 u16 vecidx; 273 void *eqc; 274 int inlen; 275 u32 *in; 276 int err; 277 int i; 278 279 /* Init CQ table */ 280 memset(cq_table, 0, sizeof(*cq_table)); 281 spin_lock_init(&cq_table->lock); 282 INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC); 283 284 eq->cons_index = 0; 285 286 err = mlx5_frag_buf_alloc_node(dev, wq_get_byte_sz(log_eq_size, log_eq_stride), 287 &eq->frag_buf, dev->priv.numa_node); 288 if (err) 289 return err; 290 291 mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc); 292 init_eq_buf(eq); 293 294 eq->irq = param->irq; 295 vecidx = mlx5_irq_get_index(eq->irq); 296 297 inlen = MLX5_ST_SZ_BYTES(create_eq_in) + 298 MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages; 299 300 in = kvzalloc(inlen, GFP_KERNEL); 301 if (!in) { 302 err = -ENOMEM; 303 goto err_buf; 304 } 305 306 pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas); 307 mlx5_fill_page_frag_array(&eq->frag_buf, pas); 308 309 MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); 310 if (!param->mask[0] && MLX5_CAP_GEN(dev, log_max_uctx)) 311 MLX5_SET(create_eq_in, in, uid, MLX5_SHARED_RESOURCE_UID); 312 313 for (i = 0; i < 4; i++) 314 MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, i, 315 param->mask[i]); 316 317 eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); 318 MLX5_SET(eqc, eqc, log_eq_size, eq->fbc.log_sz); 319 MLX5_SET(eqc, eqc, uar_page, priv->uar->index); 320 MLX5_SET(eqc, eqc, intr, vecidx); 321 MLX5_SET(eqc, eqc, log_page_size, 322 eq->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 323 324 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 325 if (err) 326 goto err_in; 327 328 eq->vecidx = vecidx; 329 eq->eqn = MLX5_GET(create_eq_out, out, eq_number); 330 eq->irqn = pci_irq_vector(dev->pdev, vecidx); 331 eq->dev = dev; 332 eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET; 333 334 err = mlx5_debug_eq_add(dev, eq); 335 if (err) 336 goto err_eq; 337 338 kvfree(in); 339 return 0; 340 341 err_eq: 342 mlx5_cmd_destroy_eq(dev, eq->eqn); 343 344 err_in: 345 kvfree(in); 346 347 err_buf: 348 mlx5_frag_buf_free(dev, &eq->frag_buf); 349 return err; 350 } 351 352 /** 353 * mlx5_eq_enable - Enable EQ for receiving EQEs 354 * @dev : Device which owns the eq 355 * @eq : EQ to enable 356 * @nb : Notifier call block 357 * 358 * Must be called after EQ is created in device. 359 * 360 * @return: 0 if no error 361 */ 362 int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq, 363 struct notifier_block *nb) 364 { 365 int err; 366 367 err = mlx5_irq_attach_nb(eq->irq, nb); 368 if (!err) 369 eq_update_ci(eq, 1); 370 371 return err; 372 } 373 EXPORT_SYMBOL(mlx5_eq_enable); 374 375 /** 376 * mlx5_eq_disable - Disable EQ for receiving EQEs 377 * @dev : Device which owns the eq 378 * @eq : EQ to disable 379 * @nb : Notifier call block 380 * 381 * Must be called before EQ is destroyed. 382 */ 383 void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq, 384 struct notifier_block *nb) 385 { 386 mlx5_irq_detach_nb(eq->irq, nb); 387 } 388 EXPORT_SYMBOL(mlx5_eq_disable); 389 390 static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) 391 { 392 int err; 393 394 mlx5_debug_eq_remove(dev, eq); 395 396 err = mlx5_cmd_destroy_eq(dev, eq->eqn); 397 if (err) 398 mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n", 399 eq->eqn); 400 401 mlx5_frag_buf_free(dev, &eq->frag_buf); 402 return err; 403 } 404 405 int mlx5_eq_add_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq) 406 { 407 struct mlx5_cq_table *table = &eq->cq_table; 408 int err; 409 410 spin_lock(&table->lock); 411 err = radix_tree_insert(&table->tree, cq->cqn, cq); 412 spin_unlock(&table->lock); 413 414 return err; 415 } 416 417 void mlx5_eq_del_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq) 418 { 419 struct mlx5_cq_table *table = &eq->cq_table; 420 struct mlx5_core_cq *tmp; 421 422 spin_lock(&table->lock); 423 tmp = radix_tree_delete(&table->tree, cq->cqn); 424 spin_unlock(&table->lock); 425 426 if (!tmp) { 427 mlx5_core_dbg(eq->dev, "cq 0x%x not found in eq 0x%x tree\n", 428 eq->eqn, cq->cqn); 429 return; 430 } 431 432 if (tmp != cq) 433 mlx5_core_dbg(eq->dev, "corruption on cqn 0x%x in eq 0x%x\n", 434 eq->eqn, cq->cqn); 435 } 436 437 int mlx5_eq_table_init(struct mlx5_core_dev *dev) 438 { 439 struct mlx5_eq_table *eq_table; 440 int i; 441 442 eq_table = kvzalloc(sizeof(*eq_table), GFP_KERNEL); 443 if (!eq_table) 444 return -ENOMEM; 445 446 dev->priv.eq_table = eq_table; 447 448 mlx5_eq_debugfs_init(dev); 449 450 mutex_init(&eq_table->lock); 451 for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++) 452 ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]); 453 454 eq_table->irq_table = mlx5_irq_table_get(dev); 455 return 0; 456 } 457 458 void mlx5_eq_table_cleanup(struct mlx5_core_dev *dev) 459 { 460 mlx5_eq_debugfs_cleanup(dev); 461 kvfree(dev->priv.eq_table); 462 } 463 464 /* Async EQs */ 465 466 static int create_async_eq(struct mlx5_core_dev *dev, 467 struct mlx5_eq *eq, struct mlx5_eq_param *param) 468 { 469 struct mlx5_eq_table *eq_table = dev->priv.eq_table; 470 int err; 471 472 mutex_lock(&eq_table->lock); 473 err = create_map_eq(dev, eq, param); 474 mutex_unlock(&eq_table->lock); 475 return err; 476 } 477 478 static int destroy_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) 479 { 480 struct mlx5_eq_table *eq_table = dev->priv.eq_table; 481 int err; 482 483 mutex_lock(&eq_table->lock); 484 err = destroy_unmap_eq(dev, eq); 485 mutex_unlock(&eq_table->lock); 486 return err; 487 } 488 489 static int cq_err_event_notifier(struct notifier_block *nb, 490 unsigned long type, void *data) 491 { 492 struct mlx5_eq_table *eqt; 493 struct mlx5_core_cq *cq; 494 struct mlx5_eqe *eqe; 495 struct mlx5_eq *eq; 496 u32 cqn; 497 498 /* type == MLX5_EVENT_TYPE_CQ_ERROR */ 499 500 eqt = mlx5_nb_cof(nb, struct mlx5_eq_table, cq_err_nb); 501 eq = &eqt->async_eq.core; 502 eqe = data; 503 504 cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff; 505 mlx5_core_warn(eq->dev, "CQ error on CQN 0x%x, syndrome 0x%x\n", 506 cqn, eqe->data.cq_err.syndrome); 507 508 cq = mlx5_eq_cq_get(eq, cqn); 509 if (unlikely(!cq)) { 510 mlx5_core_warn(eq->dev, "Async event for bogus CQ 0x%x\n", cqn); 511 return NOTIFY_OK; 512 } 513 514 if (cq->event) 515 cq->event(cq, type); 516 517 mlx5_cq_put(cq); 518 519 return NOTIFY_OK; 520 } 521 522 static void gather_user_async_events(struct mlx5_core_dev *dev, u64 mask[4]) 523 { 524 __be64 *user_unaffiliated_events; 525 __be64 *user_affiliated_events; 526 int i; 527 528 user_affiliated_events = 529 MLX5_CAP_DEV_EVENT(dev, user_affiliated_events); 530 user_unaffiliated_events = 531 MLX5_CAP_DEV_EVENT(dev, user_unaffiliated_events); 532 533 for (i = 0; i < 4; i++) 534 mask[i] |= be64_to_cpu(user_affiliated_events[i] | 535 user_unaffiliated_events[i]); 536 } 537 538 static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) 539 { 540 u64 async_event_mask = MLX5_ASYNC_EVENT_MASK; 541 542 if (MLX5_VPORT_MANAGER(dev)) 543 async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE); 544 545 if (MLX5_CAP_GEN(dev, general_notification_event)) 546 async_event_mask |= (1ull << MLX5_EVENT_TYPE_GENERAL_EVENT); 547 548 if (MLX5_CAP_GEN(dev, port_module_event)) 549 async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT); 550 else 551 mlx5_core_dbg(dev, "port_module_event is not set\n"); 552 553 if (MLX5_PPS_CAP(dev)) 554 async_event_mask |= (1ull << MLX5_EVENT_TYPE_PPS_EVENT); 555 556 if (MLX5_CAP_GEN(dev, fpga)) 557 async_event_mask |= (1ull << MLX5_EVENT_TYPE_FPGA_ERROR) | 558 (1ull << MLX5_EVENT_TYPE_FPGA_QP_ERROR); 559 if (MLX5_CAP_GEN_MAX(dev, dct)) 560 async_event_mask |= (1ull << MLX5_EVENT_TYPE_DCT_DRAINED); 561 562 if (MLX5_CAP_GEN(dev, temp_warn_event)) 563 async_event_mask |= (1ull << MLX5_EVENT_TYPE_TEMP_WARN_EVENT); 564 565 if (MLX5_CAP_MCAM_REG(dev, tracer_registers)) 566 async_event_mask |= (1ull << MLX5_EVENT_TYPE_DEVICE_TRACER); 567 568 if (MLX5_CAP_GEN(dev, max_num_of_monitor_counters)) 569 async_event_mask |= (1ull << MLX5_EVENT_TYPE_MONITOR_COUNTER); 570 571 if (mlx5_eswitch_is_funcs_handler(dev)) 572 async_event_mask |= 573 (1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED); 574 575 if (MLX5_CAP_GEN_MAX(dev, vhca_state)) 576 async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE); 577 578 mask[0] = async_event_mask; 579 580 if (MLX5_CAP_GEN(dev, event_cap)) 581 gather_user_async_events(dev, mask); 582 } 583 584 static int 585 setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq, 586 struct mlx5_eq_param *param, const char *name) 587 { 588 int err; 589 590 eq->irq_nb.notifier_call = mlx5_eq_async_int; 591 spin_lock_init(&eq->lock); 592 593 err = create_async_eq(dev, &eq->core, param); 594 if (err) { 595 mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err); 596 return err; 597 } 598 err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb); 599 if (err) { 600 mlx5_core_warn(dev, "failed to enable %s EQ %d\n", name, err); 601 destroy_async_eq(dev, &eq->core); 602 } 603 return err; 604 } 605 606 static void cleanup_async_eq(struct mlx5_core_dev *dev, 607 struct mlx5_eq_async *eq, const char *name) 608 { 609 int err; 610 611 mlx5_eq_disable(dev, &eq->core, &eq->irq_nb); 612 err = destroy_async_eq(dev, &eq->core); 613 if (err) 614 mlx5_core_err(dev, "failed to destroy %s eq, err(%d)\n", 615 name, err); 616 } 617 618 static u16 async_eq_depth_devlink_param_get(struct mlx5_core_dev *dev) 619 { 620 struct devlink *devlink = priv_to_devlink(dev); 621 union devlink_param_value val; 622 int err; 623 624 err = devlink_param_driverinit_value_get(devlink, 625 DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE, 626 &val); 627 if (!err) 628 return val.vu32; 629 mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err); 630 return MLX5_NUM_ASYNC_EQE; 631 } 632 static int create_async_eqs(struct mlx5_core_dev *dev) 633 { 634 struct mlx5_eq_table *table = dev->priv.eq_table; 635 struct mlx5_eq_param param = {}; 636 int err; 637 638 /* All the async_eqs are using single IRQ, request one IRQ and share its 639 * index among all the async_eqs of this device. 640 */ 641 table->ctrl_irq = mlx5_ctrl_irq_request(dev); 642 if (IS_ERR(table->ctrl_irq)) 643 return PTR_ERR(table->ctrl_irq); 644 645 MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR); 646 mlx5_eq_notifier_register(dev, &table->cq_err_nb); 647 648 param = (struct mlx5_eq_param) { 649 .irq = table->ctrl_irq, 650 .nent = MLX5_NUM_CMD_EQE, 651 .mask[0] = 1ull << MLX5_EVENT_TYPE_CMD, 652 }; 653 mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_CREATE_EQ); 654 err = setup_async_eq(dev, &table->cmd_eq, ¶m, "cmd"); 655 if (err) 656 goto err1; 657 658 mlx5_cmd_use_events(dev); 659 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); 660 661 param = (struct mlx5_eq_param) { 662 .irq = table->ctrl_irq, 663 .nent = async_eq_depth_devlink_param_get(dev), 664 }; 665 666 gather_async_events_mask(dev, param.mask); 667 err = setup_async_eq(dev, &table->async_eq, ¶m, "async"); 668 if (err) 669 goto err2; 670 671 param = (struct mlx5_eq_param) { 672 .irq = table->ctrl_irq, 673 .nent = /* TODO: sriov max_vf + */ 1, 674 .mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST, 675 }; 676 677 err = setup_async_eq(dev, &table->pages_eq, ¶m, "pages"); 678 if (err) 679 goto err3; 680 681 return 0; 682 683 err3: 684 cleanup_async_eq(dev, &table->async_eq, "async"); 685 err2: 686 mlx5_cmd_use_polling(dev); 687 cleanup_async_eq(dev, &table->cmd_eq, "cmd"); 688 err1: 689 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); 690 mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); 691 mlx5_ctrl_irq_release(table->ctrl_irq); 692 return err; 693 } 694 695 static void destroy_async_eqs(struct mlx5_core_dev *dev) 696 { 697 struct mlx5_eq_table *table = dev->priv.eq_table; 698 699 cleanup_async_eq(dev, &table->pages_eq, "pages"); 700 cleanup_async_eq(dev, &table->async_eq, "async"); 701 mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_DESTROY_EQ); 702 mlx5_cmd_use_polling(dev); 703 cleanup_async_eq(dev, &table->cmd_eq, "cmd"); 704 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); 705 mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); 706 mlx5_ctrl_irq_release(table->ctrl_irq); 707 } 708 709 struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev) 710 { 711 return &dev->priv.eq_table->async_eq.core; 712 } 713 714 void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev) 715 { 716 synchronize_irq(dev->priv.eq_table->async_eq.core.irqn); 717 } 718 719 void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev) 720 { 721 synchronize_irq(dev->priv.eq_table->cmd_eq.core.irqn); 722 } 723 724 /* Generic EQ API for mlx5_core consumers 725 * Needed For RDMA ODP EQ for now 726 */ 727 struct mlx5_eq * 728 mlx5_eq_create_generic(struct mlx5_core_dev *dev, 729 struct mlx5_eq_param *param) 730 { 731 struct mlx5_eq *eq = kvzalloc(sizeof(*eq), GFP_KERNEL); 732 int err; 733 734 if (!eq) 735 return ERR_PTR(-ENOMEM); 736 737 param->irq = dev->priv.eq_table->ctrl_irq; 738 err = create_async_eq(dev, eq, param); 739 if (err) { 740 kvfree(eq); 741 eq = ERR_PTR(err); 742 } 743 744 return eq; 745 } 746 EXPORT_SYMBOL(mlx5_eq_create_generic); 747 748 int mlx5_eq_destroy_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq) 749 { 750 int err; 751 752 if (IS_ERR(eq)) 753 return -EINVAL; 754 755 err = destroy_async_eq(dev, eq); 756 if (err) 757 goto out; 758 759 kvfree(eq); 760 out: 761 return err; 762 } 763 EXPORT_SYMBOL(mlx5_eq_destroy_generic); 764 765 struct mlx5_eqe *mlx5_eq_get_eqe(struct mlx5_eq *eq, u32 cc) 766 { 767 u32 ci = eq->cons_index + cc; 768 u32 nent = eq_get_size(eq); 769 struct mlx5_eqe *eqe; 770 771 eqe = get_eqe(eq, ci & (nent - 1)); 772 eqe = ((eqe->owner & 1) ^ !!(ci & nent)) ? NULL : eqe; 773 /* Make sure we read EQ entry contents after we've 774 * checked the ownership bit. 775 */ 776 if (eqe) 777 dma_rmb(); 778 779 return eqe; 780 } 781 EXPORT_SYMBOL(mlx5_eq_get_eqe); 782 783 void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm) 784 { 785 __be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2); 786 u32 val; 787 788 eq->cons_index += cc; 789 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24); 790 791 __raw_writel((__force u32)cpu_to_be32(val), addr); 792 /* We still want ordering, just not swabbing, so add a barrier */ 793 wmb(); 794 } 795 EXPORT_SYMBOL(mlx5_eq_update_ci); 796 797 static void comp_irqs_release(struct mlx5_core_dev *dev) 798 { 799 struct mlx5_eq_table *table = dev->priv.eq_table; 800 801 if (mlx5_core_is_sf(dev)) 802 mlx5_irq_affinity_irqs_release(dev, table->comp_irqs, table->num_comp_eqs); 803 else 804 mlx5_irqs_release_vectors(table->comp_irqs, table->num_comp_eqs); 805 kfree(table->comp_irqs); 806 } 807 808 static int comp_irqs_request(struct mlx5_core_dev *dev) 809 { 810 struct mlx5_eq_table *table = dev->priv.eq_table; 811 int ncomp_eqs = table->num_comp_eqs; 812 u16 *cpus; 813 int ret; 814 int i; 815 816 ncomp_eqs = table->num_comp_eqs; 817 table->comp_irqs = kcalloc(ncomp_eqs, sizeof(*table->comp_irqs), GFP_KERNEL); 818 if (!table->comp_irqs) 819 return -ENOMEM; 820 if (mlx5_core_is_sf(dev)) { 821 ret = mlx5_irq_affinity_irqs_request_auto(dev, ncomp_eqs, table->comp_irqs); 822 if (ret < 0) 823 goto free_irqs; 824 return ret; 825 } 826 827 cpus = kcalloc(ncomp_eqs, sizeof(*cpus), GFP_KERNEL); 828 if (!cpus) { 829 ret = -ENOMEM; 830 goto free_irqs; 831 } 832 for (i = 0; i < ncomp_eqs; i++) 833 cpus[i] = cpumask_local_spread(i, dev->priv.numa_node); 834 ret = mlx5_irqs_request_vectors(dev, cpus, ncomp_eqs, table->comp_irqs); 835 kfree(cpus); 836 if (ret < 0) 837 goto free_irqs; 838 return ret; 839 840 free_irqs: 841 kfree(table->comp_irqs); 842 return ret; 843 } 844 845 static void destroy_comp_eqs(struct mlx5_core_dev *dev) 846 { 847 struct mlx5_eq_table *table = dev->priv.eq_table; 848 struct mlx5_eq_comp *eq, *n; 849 850 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 851 list_del(&eq->list); 852 mlx5_eq_disable(dev, &eq->core, &eq->irq_nb); 853 if (destroy_unmap_eq(dev, &eq->core)) 854 mlx5_core_warn(dev, "failed to destroy comp EQ 0x%x\n", 855 eq->core.eqn); 856 tasklet_disable(&eq->tasklet_ctx.task); 857 kfree(eq); 858 } 859 comp_irqs_release(dev); 860 } 861 862 static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev) 863 { 864 struct devlink *devlink = priv_to_devlink(dev); 865 union devlink_param_value val; 866 int err; 867 868 err = devlink_param_driverinit_value_get(devlink, 869 DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE, 870 &val); 871 if (!err) 872 return val.vu32; 873 mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err); 874 return MLX5_COMP_EQ_SIZE; 875 } 876 877 static int create_comp_eqs(struct mlx5_core_dev *dev) 878 { 879 struct mlx5_eq_table *table = dev->priv.eq_table; 880 struct mlx5_eq_comp *eq; 881 int ncomp_eqs; 882 int nent; 883 int err; 884 int i; 885 886 ncomp_eqs = comp_irqs_request(dev); 887 if (ncomp_eqs < 0) 888 return ncomp_eqs; 889 INIT_LIST_HEAD(&table->comp_eqs_list); 890 nent = comp_eq_depth_devlink_param_get(dev); 891 for (i = 0; i < ncomp_eqs; i++) { 892 struct mlx5_eq_param param = {}; 893 894 eq = kzalloc(sizeof(*eq), GFP_KERNEL); 895 if (!eq) { 896 err = -ENOMEM; 897 goto clean; 898 } 899 900 INIT_LIST_HEAD(&eq->tasklet_ctx.list); 901 INIT_LIST_HEAD(&eq->tasklet_ctx.process_list); 902 spin_lock_init(&eq->tasklet_ctx.lock); 903 tasklet_setup(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb); 904 905 eq->irq_nb.notifier_call = mlx5_eq_comp_int; 906 param = (struct mlx5_eq_param) { 907 .irq = table->comp_irqs[i], 908 .nent = nent, 909 }; 910 911 err = create_map_eq(dev, &eq->core, ¶m); 912 if (err) 913 goto clean_eq; 914 err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb); 915 if (err) { 916 destroy_unmap_eq(dev, &eq->core); 917 goto clean_eq; 918 } 919 920 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn); 921 /* add tail, to keep the list ordered, for mlx5_vector2eqn to work */ 922 list_add_tail(&eq->list, &table->comp_eqs_list); 923 } 924 925 table->num_comp_eqs = ncomp_eqs; 926 return 0; 927 928 clean_eq: 929 kfree(eq); 930 clean: 931 destroy_comp_eqs(dev); 932 return err; 933 } 934 935 static int vector2eqnirqn(struct mlx5_core_dev *dev, int vector, int *eqn, 936 unsigned int *irqn) 937 { 938 struct mlx5_eq_table *table = dev->priv.eq_table; 939 struct mlx5_eq_comp *eq, *n; 940 int err = -ENOENT; 941 int i = 0; 942 943 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 944 if (i++ == vector) { 945 if (irqn) 946 *irqn = eq->core.irqn; 947 if (eqn) 948 *eqn = eq->core.eqn; 949 err = 0; 950 break; 951 } 952 } 953 954 return err; 955 } 956 957 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn) 958 { 959 return vector2eqnirqn(dev, vector, eqn, NULL); 960 } 961 EXPORT_SYMBOL(mlx5_vector2eqn); 962 963 int mlx5_vector2irqn(struct mlx5_core_dev *dev, int vector, unsigned int *irqn) 964 { 965 return vector2eqnirqn(dev, vector, NULL, irqn); 966 } 967 968 unsigned int mlx5_comp_vectors_count(struct mlx5_core_dev *dev) 969 { 970 return dev->priv.eq_table->num_comp_eqs; 971 } 972 EXPORT_SYMBOL(mlx5_comp_vectors_count); 973 974 struct cpumask * 975 mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector) 976 { 977 struct mlx5_eq_table *table = dev->priv.eq_table; 978 struct mlx5_eq_comp *eq, *n; 979 int i = 0; 980 981 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 982 if (i++ == vector) 983 break; 984 } 985 986 return mlx5_irq_get_affinity_mask(eq->core.irq); 987 } 988 EXPORT_SYMBOL(mlx5_comp_irq_get_affinity_mask); 989 990 #ifdef CONFIG_RFS_ACCEL 991 struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev) 992 { 993 return dev->priv.eq_table->rmap; 994 } 995 #endif 996 997 struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn) 998 { 999 struct mlx5_eq_table *table = dev->priv.eq_table; 1000 struct mlx5_eq_comp *eq; 1001 1002 list_for_each_entry(eq, &table->comp_eqs_list, list) { 1003 if (eq->core.eqn == eqn) 1004 return eq; 1005 } 1006 1007 return ERR_PTR(-ENOENT); 1008 } 1009 1010 static void clear_rmap(struct mlx5_core_dev *dev) 1011 { 1012 #ifdef CONFIG_RFS_ACCEL 1013 struct mlx5_eq_table *eq_table = dev->priv.eq_table; 1014 1015 free_irq_cpu_rmap(eq_table->rmap); 1016 #endif 1017 } 1018 1019 static int set_rmap(struct mlx5_core_dev *mdev) 1020 { 1021 int err = 0; 1022 #ifdef CONFIG_RFS_ACCEL 1023 struct mlx5_eq_table *eq_table = mdev->priv.eq_table; 1024 int vecidx; 1025 1026 eq_table->rmap = alloc_irq_cpu_rmap(eq_table->num_comp_eqs); 1027 if (!eq_table->rmap) { 1028 err = -ENOMEM; 1029 mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err); 1030 goto err_out; 1031 } 1032 1033 for (vecidx = 0; vecidx < eq_table->num_comp_eqs; vecidx++) { 1034 err = irq_cpu_rmap_add(eq_table->rmap, 1035 pci_irq_vector(mdev->pdev, vecidx)); 1036 if (err) { 1037 mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d", 1038 err); 1039 goto err_irq_cpu_rmap_add; 1040 } 1041 } 1042 return 0; 1043 1044 err_irq_cpu_rmap_add: 1045 clear_rmap(mdev); 1046 err_out: 1047 #endif 1048 return err; 1049 } 1050 1051 /* This function should only be called after mlx5_cmd_force_teardown_hca */ 1052 void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev) 1053 { 1054 struct mlx5_eq_table *table = dev->priv.eq_table; 1055 1056 mutex_lock(&table->lock); /* sync with create/destroy_async_eq */ 1057 if (!mlx5_core_is_sf(dev)) 1058 clear_rmap(dev); 1059 mlx5_irq_table_destroy(dev); 1060 mutex_unlock(&table->lock); 1061 } 1062 1063 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1064 #define MLX5_MAX_ASYNC_EQS 4 1065 #else 1066 #define MLX5_MAX_ASYNC_EQS 3 1067 #endif 1068 1069 int mlx5_eq_table_create(struct mlx5_core_dev *dev) 1070 { 1071 struct mlx5_eq_table *eq_table = dev->priv.eq_table; 1072 int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ? 1073 MLX5_CAP_GEN(dev, max_num_eqs) : 1074 1 << MLX5_CAP_GEN(dev, log_max_eq); 1075 int max_eqs_sf; 1076 int err; 1077 1078 eq_table->num_comp_eqs = 1079 min_t(int, 1080 mlx5_irq_table_get_num_comp(eq_table->irq_table), 1081 num_eqs - MLX5_MAX_ASYNC_EQS); 1082 if (mlx5_core_is_sf(dev)) { 1083 max_eqs_sf = min_t(int, MLX5_COMP_EQS_PER_SF, 1084 mlx5_irq_table_get_sfs_vec(eq_table->irq_table)); 1085 eq_table->num_comp_eqs = min_t(int, eq_table->num_comp_eqs, 1086 max_eqs_sf); 1087 } 1088 1089 err = create_async_eqs(dev); 1090 if (err) { 1091 mlx5_core_err(dev, "Failed to create async EQs\n"); 1092 goto err_async_eqs; 1093 } 1094 1095 if (!mlx5_core_is_sf(dev)) { 1096 /* rmap is a mapping between irq number and queue number. 1097 * each irq can be assign only to a single rmap. 1098 * since SFs share IRQs, rmap mapping cannot function correctly 1099 * for irqs that are shared for different core/netdev RX rings. 1100 * Hence we don't allow netdev rmap for SFs 1101 */ 1102 err = set_rmap(dev); 1103 if (err) 1104 goto err_rmap; 1105 } 1106 1107 err = create_comp_eqs(dev); 1108 if (err) { 1109 mlx5_core_err(dev, "Failed to create completion EQs\n"); 1110 goto err_comp_eqs; 1111 } 1112 1113 return 0; 1114 err_comp_eqs: 1115 if (!mlx5_core_is_sf(dev)) 1116 clear_rmap(dev); 1117 err_rmap: 1118 destroy_async_eqs(dev); 1119 err_async_eqs: 1120 return err; 1121 } 1122 1123 void mlx5_eq_table_destroy(struct mlx5_core_dev *dev) 1124 { 1125 if (!mlx5_core_is_sf(dev)) 1126 clear_rmap(dev); 1127 destroy_comp_eqs(dev); 1128 destroy_async_eqs(dev); 1129 } 1130 1131 int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb) 1132 { 1133 struct mlx5_eq_table *eqt = dev->priv.eq_table; 1134 1135 return atomic_notifier_chain_register(&eqt->nh[nb->event_type], &nb->nb); 1136 } 1137 EXPORT_SYMBOL(mlx5_eq_notifier_register); 1138 1139 int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb) 1140 { 1141 struct mlx5_eq_table *eqt = dev->priv.eq_table; 1142 1143 return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb); 1144 } 1145 EXPORT_SYMBOL(mlx5_eq_notifier_unregister); 1146