1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2021, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/interrupt.h> 7 #include <linux/notifier.h> 8 #include <linux/mlx5/driver.h> 9 #include <linux/mlx5/vport.h> 10 #include <linux/mlx5/eq.h> 11 #ifdef CONFIG_RFS_ACCEL 12 #include <linux/cpu_rmap.h> 13 #endif 14 #include "mlx5_core.h" 15 #include "lib/eq.h" 16 #include "fpga/core.h" 17 #include "eswitch.h" 18 #include "lib/clock.h" 19 #include "diag/fw_tracer.h" 20 #include "mlx5_irq.h" 21 #include "pci_irq.h" 22 #include "devlink.h" 23 #include "en_accel/ipsec.h" 24 25 enum { 26 MLX5_EQE_OWNER_INIT_VAL = 0x1, 27 }; 28 29 enum { 30 MLX5_EQ_STATE_ARMED = 0x9, 31 MLX5_EQ_STATE_FIRED = 0xa, 32 MLX5_EQ_STATE_ALWAYS_ARMED = 0xb, 33 }; 34 35 enum { 36 MLX5_EQ_DOORBEL_OFFSET = 0x40, 37 }; 38 39 /* budget must be smaller than MLX5_NUM_SPARE_EQE to guarantee that we update 40 * the ci before we polled all the entries in the EQ. MLX5_NUM_SPARE_EQE is 41 * used to set the EQ size, budget must be smaller than the EQ size. 42 */ 43 enum { 44 MLX5_EQ_POLLING_BUDGET = 128, 45 }; 46 47 static_assert(MLX5_EQ_POLLING_BUDGET <= MLX5_NUM_SPARE_EQE); 48 49 struct mlx5_eq_table { 50 struct list_head comp_eqs_list; 51 struct mlx5_eq_async pages_eq; 52 struct mlx5_eq_async cmd_eq; 53 struct mlx5_eq_async async_eq; 54 55 struct atomic_notifier_head nh[MLX5_EVENT_TYPE_MAX]; 56 57 /* Since CQ DB is stored in async_eq */ 58 struct mlx5_nb cq_err_nb; 59 60 struct mutex lock; /* sync async eqs creations */ 61 int num_comp_eqs; 62 struct mlx5_irq_table *irq_table; 63 struct mlx5_irq **comp_irqs; 64 struct mlx5_irq *ctrl_irq; 65 struct cpu_rmap *rmap; 66 }; 67 68 #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG) | \ 69 (1ull << MLX5_EVENT_TYPE_COMM_EST) | \ 70 (1ull << MLX5_EVENT_TYPE_SQ_DRAINED) | \ 71 (1ull << MLX5_EVENT_TYPE_CQ_ERROR) | \ 72 (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR) | \ 73 (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED) | \ 74 (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ 75 (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR) | \ 76 (1ull << MLX5_EVENT_TYPE_PORT_CHANGE) | \ 77 (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR) | \ 78 (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE) | \ 79 (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT)) 80 81 static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn) 82 { 83 u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {}; 84 85 MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ); 86 MLX5_SET(destroy_eq_in, in, eq_number, eqn); 87 return mlx5_cmd_exec_in(dev, destroy_eq, in); 88 } 89 90 /* caller must eventually call mlx5_cq_put on the returned cq */ 91 static struct mlx5_core_cq *mlx5_eq_cq_get(struct mlx5_eq *eq, u32 cqn) 92 { 93 struct mlx5_cq_table *table = &eq->cq_table; 94 struct mlx5_core_cq *cq = NULL; 95 96 rcu_read_lock(); 97 cq = radix_tree_lookup(&table->tree, cqn); 98 if (likely(cq)) 99 mlx5_cq_hold(cq); 100 rcu_read_unlock(); 101 102 return cq; 103 } 104 105 static int mlx5_eq_comp_int(struct notifier_block *nb, 106 __always_unused unsigned long action, 107 __always_unused void *data) 108 { 109 struct mlx5_eq_comp *eq_comp = 110 container_of(nb, struct mlx5_eq_comp, irq_nb); 111 struct mlx5_eq *eq = &eq_comp->core; 112 struct mlx5_eqe *eqe; 113 int num_eqes = 0; 114 u32 cqn = -1; 115 116 eqe = next_eqe_sw(eq); 117 if (!eqe) 118 goto out; 119 120 do { 121 struct mlx5_core_cq *cq; 122 123 /* Make sure we read EQ entry contents after we've 124 * checked the ownership bit. 125 */ 126 dma_rmb(); 127 /* Assume (eqe->type) is always MLX5_EVENT_TYPE_COMP */ 128 cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff; 129 130 cq = mlx5_eq_cq_get(eq, cqn); 131 if (likely(cq)) { 132 ++cq->arm_sn; 133 cq->comp(cq, eqe); 134 mlx5_cq_put(cq); 135 } else { 136 dev_dbg_ratelimited(eq->dev->device, 137 "Completion event for bogus CQ 0x%x\n", cqn); 138 } 139 140 ++eq->cons_index; 141 142 } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq))); 143 144 out: 145 eq_update_ci(eq, 1); 146 147 if (cqn != -1) 148 tasklet_schedule(&eq_comp->tasklet_ctx.task); 149 150 return 0; 151 } 152 153 /* Some architectures don't latch interrupts when they are disabled, so using 154 * mlx5_eq_poll_irq_disabled could end up losing interrupts while trying to 155 * avoid losing them. It is not recommended to use it, unless this is the last 156 * resort. 157 */ 158 u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq) 159 { 160 u32 count_eqe; 161 162 disable_irq(eq->core.irqn); 163 count_eqe = eq->core.cons_index; 164 mlx5_eq_comp_int(&eq->irq_nb, 0, NULL); 165 count_eqe = eq->core.cons_index - count_eqe; 166 enable_irq(eq->core.irqn); 167 168 return count_eqe; 169 } 170 171 static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, bool recovery, 172 unsigned long *flags) 173 __acquires(&eq->lock) 174 { 175 if (!recovery) 176 spin_lock(&eq->lock); 177 else 178 spin_lock_irqsave(&eq->lock, *flags); 179 } 180 181 static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, bool recovery, 182 unsigned long *flags) 183 __releases(&eq->lock) 184 { 185 if (!recovery) 186 spin_unlock(&eq->lock); 187 else 188 spin_unlock_irqrestore(&eq->lock, *flags); 189 } 190 191 enum async_eq_nb_action { 192 ASYNC_EQ_IRQ_HANDLER = 0, 193 ASYNC_EQ_RECOVER = 1, 194 }; 195 196 static int mlx5_eq_async_int(struct notifier_block *nb, 197 unsigned long action, void *data) 198 { 199 struct mlx5_eq_async *eq_async = 200 container_of(nb, struct mlx5_eq_async, irq_nb); 201 struct mlx5_eq *eq = &eq_async->core; 202 struct mlx5_eq_table *eqt; 203 struct mlx5_core_dev *dev; 204 struct mlx5_eqe *eqe; 205 unsigned long flags; 206 int num_eqes = 0; 207 bool recovery; 208 209 dev = eq->dev; 210 eqt = dev->priv.eq_table; 211 212 recovery = action == ASYNC_EQ_RECOVER; 213 mlx5_eq_async_int_lock(eq_async, recovery, &flags); 214 215 eqe = next_eqe_sw(eq); 216 if (!eqe) 217 goto out; 218 219 do { 220 /* 221 * Make sure we read EQ entry contents after we've 222 * checked the ownership bit. 223 */ 224 dma_rmb(); 225 226 atomic_notifier_call_chain(&eqt->nh[eqe->type], eqe->type, eqe); 227 atomic_notifier_call_chain(&eqt->nh[MLX5_EVENT_TYPE_NOTIFY_ANY], eqe->type, eqe); 228 229 ++eq->cons_index; 230 231 } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq))); 232 233 out: 234 eq_update_ci(eq, 1); 235 mlx5_eq_async_int_unlock(eq_async, recovery, &flags); 236 237 return unlikely(recovery) ? num_eqes : 0; 238 } 239 240 void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev) 241 { 242 struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq; 243 int eqes; 244 245 eqes = mlx5_eq_async_int(&eq->irq_nb, ASYNC_EQ_RECOVER, NULL); 246 if (eqes) 247 mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes); 248 } 249 250 static void init_eq_buf(struct mlx5_eq *eq) 251 { 252 struct mlx5_eqe *eqe; 253 int i; 254 255 for (i = 0; i < eq_get_size(eq); i++) { 256 eqe = get_eqe(eq, i); 257 eqe->owner = MLX5_EQE_OWNER_INIT_VAL; 258 } 259 } 260 261 static int 262 create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, 263 struct mlx5_eq_param *param) 264 { 265 u8 log_eq_size = order_base_2(param->nent + MLX5_NUM_SPARE_EQE); 266 struct mlx5_cq_table *cq_table = &eq->cq_table; 267 u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0}; 268 u8 log_eq_stride = ilog2(MLX5_EQE_SIZE); 269 struct mlx5_priv *priv = &dev->priv; 270 __be64 *pas; 271 u16 vecidx; 272 void *eqc; 273 int inlen; 274 u32 *in; 275 int err; 276 int i; 277 278 /* Init CQ table */ 279 memset(cq_table, 0, sizeof(*cq_table)); 280 spin_lock_init(&cq_table->lock); 281 INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC); 282 283 eq->cons_index = 0; 284 285 err = mlx5_frag_buf_alloc_node(dev, wq_get_byte_sz(log_eq_size, log_eq_stride), 286 &eq->frag_buf, dev->priv.numa_node); 287 if (err) 288 return err; 289 290 mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc); 291 init_eq_buf(eq); 292 293 eq->irq = param->irq; 294 vecidx = mlx5_irq_get_index(eq->irq); 295 296 inlen = MLX5_ST_SZ_BYTES(create_eq_in) + 297 MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages; 298 299 in = kvzalloc(inlen, GFP_KERNEL); 300 if (!in) { 301 err = -ENOMEM; 302 goto err_buf; 303 } 304 305 pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas); 306 mlx5_fill_page_frag_array(&eq->frag_buf, pas); 307 308 MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ); 309 if (!param->mask[0] && MLX5_CAP_GEN(dev, log_max_uctx)) 310 MLX5_SET(create_eq_in, in, uid, MLX5_SHARED_RESOURCE_UID); 311 312 for (i = 0; i < 4; i++) 313 MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, i, 314 param->mask[i]); 315 316 eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry); 317 MLX5_SET(eqc, eqc, log_eq_size, eq->fbc.log_sz); 318 MLX5_SET(eqc, eqc, uar_page, priv->uar->index); 319 MLX5_SET(eqc, eqc, intr, vecidx); 320 MLX5_SET(eqc, eqc, log_page_size, 321 eq->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 322 323 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 324 if (err) 325 goto err_in; 326 327 eq->vecidx = vecidx; 328 eq->eqn = MLX5_GET(create_eq_out, out, eq_number); 329 eq->irqn = pci_irq_vector(dev->pdev, vecidx); 330 eq->dev = dev; 331 eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET; 332 333 err = mlx5_debug_eq_add(dev, eq); 334 if (err) 335 goto err_eq; 336 337 kvfree(in); 338 return 0; 339 340 err_eq: 341 mlx5_cmd_destroy_eq(dev, eq->eqn); 342 343 err_in: 344 kvfree(in); 345 346 err_buf: 347 mlx5_frag_buf_free(dev, &eq->frag_buf); 348 return err; 349 } 350 351 /** 352 * mlx5_eq_enable - Enable EQ for receiving EQEs 353 * @dev : Device which owns the eq 354 * @eq : EQ to enable 355 * @nb : Notifier call block 356 * 357 * Must be called after EQ is created in device. 358 * 359 * @return: 0 if no error 360 */ 361 int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq, 362 struct notifier_block *nb) 363 { 364 int err; 365 366 err = mlx5_irq_attach_nb(eq->irq, nb); 367 if (!err) 368 eq_update_ci(eq, 1); 369 370 return err; 371 } 372 EXPORT_SYMBOL(mlx5_eq_enable); 373 374 /** 375 * mlx5_eq_disable - Disable EQ for receiving EQEs 376 * @dev : Device which owns the eq 377 * @eq : EQ to disable 378 * @nb : Notifier call block 379 * 380 * Must be called before EQ is destroyed. 381 */ 382 void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq, 383 struct notifier_block *nb) 384 { 385 mlx5_irq_detach_nb(eq->irq, nb); 386 } 387 EXPORT_SYMBOL(mlx5_eq_disable); 388 389 static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) 390 { 391 int err; 392 393 mlx5_debug_eq_remove(dev, eq); 394 395 err = mlx5_cmd_destroy_eq(dev, eq->eqn); 396 if (err) 397 mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n", 398 eq->eqn); 399 400 mlx5_frag_buf_free(dev, &eq->frag_buf); 401 return err; 402 } 403 404 int mlx5_eq_add_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq) 405 { 406 struct mlx5_cq_table *table = &eq->cq_table; 407 int err; 408 409 spin_lock(&table->lock); 410 err = radix_tree_insert(&table->tree, cq->cqn, cq); 411 spin_unlock(&table->lock); 412 413 return err; 414 } 415 416 void mlx5_eq_del_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq) 417 { 418 struct mlx5_cq_table *table = &eq->cq_table; 419 struct mlx5_core_cq *tmp; 420 421 spin_lock(&table->lock); 422 tmp = radix_tree_delete(&table->tree, cq->cqn); 423 spin_unlock(&table->lock); 424 425 if (!tmp) { 426 mlx5_core_dbg(eq->dev, "cq 0x%x not found in eq 0x%x tree\n", 427 eq->eqn, cq->cqn); 428 return; 429 } 430 431 if (tmp != cq) 432 mlx5_core_dbg(eq->dev, "corruption on cqn 0x%x in eq 0x%x\n", 433 eq->eqn, cq->cqn); 434 } 435 436 int mlx5_eq_table_init(struct mlx5_core_dev *dev) 437 { 438 struct mlx5_eq_table *eq_table; 439 int i; 440 441 eq_table = kvzalloc_node(sizeof(*eq_table), GFP_KERNEL, 442 dev->priv.numa_node); 443 if (!eq_table) 444 return -ENOMEM; 445 446 dev->priv.eq_table = eq_table; 447 448 mlx5_eq_debugfs_init(dev); 449 450 mutex_init(&eq_table->lock); 451 for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++) 452 ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]); 453 454 eq_table->irq_table = mlx5_irq_table_get(dev); 455 return 0; 456 } 457 458 void mlx5_eq_table_cleanup(struct mlx5_core_dev *dev) 459 { 460 mlx5_eq_debugfs_cleanup(dev); 461 kvfree(dev->priv.eq_table); 462 } 463 464 /* Async EQs */ 465 466 static int create_async_eq(struct mlx5_core_dev *dev, 467 struct mlx5_eq *eq, struct mlx5_eq_param *param) 468 { 469 struct mlx5_eq_table *eq_table = dev->priv.eq_table; 470 int err; 471 472 mutex_lock(&eq_table->lock); 473 err = create_map_eq(dev, eq, param); 474 mutex_unlock(&eq_table->lock); 475 return err; 476 } 477 478 static int destroy_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) 479 { 480 struct mlx5_eq_table *eq_table = dev->priv.eq_table; 481 int err; 482 483 mutex_lock(&eq_table->lock); 484 err = destroy_unmap_eq(dev, eq); 485 mutex_unlock(&eq_table->lock); 486 return err; 487 } 488 489 static int cq_err_event_notifier(struct notifier_block *nb, 490 unsigned long type, void *data) 491 { 492 struct mlx5_eq_table *eqt; 493 struct mlx5_core_cq *cq; 494 struct mlx5_eqe *eqe; 495 struct mlx5_eq *eq; 496 u32 cqn; 497 498 /* type == MLX5_EVENT_TYPE_CQ_ERROR */ 499 500 eqt = mlx5_nb_cof(nb, struct mlx5_eq_table, cq_err_nb); 501 eq = &eqt->async_eq.core; 502 eqe = data; 503 504 cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff; 505 mlx5_core_warn(eq->dev, "CQ error on CQN 0x%x, syndrome 0x%x\n", 506 cqn, eqe->data.cq_err.syndrome); 507 508 cq = mlx5_eq_cq_get(eq, cqn); 509 if (unlikely(!cq)) { 510 mlx5_core_warn(eq->dev, "Async event for bogus CQ 0x%x\n", cqn); 511 return NOTIFY_OK; 512 } 513 514 if (cq->event) 515 cq->event(cq, type); 516 517 mlx5_cq_put(cq); 518 519 return NOTIFY_OK; 520 } 521 522 static void gather_user_async_events(struct mlx5_core_dev *dev, u64 mask[4]) 523 { 524 __be64 *user_unaffiliated_events; 525 __be64 *user_affiliated_events; 526 int i; 527 528 user_affiliated_events = 529 MLX5_CAP_DEV_EVENT(dev, user_affiliated_events); 530 user_unaffiliated_events = 531 MLX5_CAP_DEV_EVENT(dev, user_unaffiliated_events); 532 533 for (i = 0; i < 4; i++) 534 mask[i] |= be64_to_cpu(user_affiliated_events[i] | 535 user_unaffiliated_events[i]); 536 } 537 538 static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) 539 { 540 u64 async_event_mask = MLX5_ASYNC_EVENT_MASK; 541 542 if (MLX5_VPORT_MANAGER(dev)) 543 async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE); 544 545 if (MLX5_CAP_GEN(dev, general_notification_event)) 546 async_event_mask |= (1ull << MLX5_EVENT_TYPE_GENERAL_EVENT); 547 548 if (MLX5_CAP_GEN(dev, port_module_event)) 549 async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT); 550 else 551 mlx5_core_dbg(dev, "port_module_event is not set\n"); 552 553 if (MLX5_PPS_CAP(dev)) 554 async_event_mask |= (1ull << MLX5_EVENT_TYPE_PPS_EVENT); 555 556 if (MLX5_CAP_GEN(dev, fpga)) 557 async_event_mask |= (1ull << MLX5_EVENT_TYPE_FPGA_ERROR) | 558 (1ull << MLX5_EVENT_TYPE_FPGA_QP_ERROR); 559 if (MLX5_CAP_GEN_MAX(dev, dct)) 560 async_event_mask |= (1ull << MLX5_EVENT_TYPE_DCT_DRAINED); 561 562 if (MLX5_CAP_GEN(dev, temp_warn_event)) 563 async_event_mask |= (1ull << MLX5_EVENT_TYPE_TEMP_WARN_EVENT); 564 565 if (MLX5_CAP_MCAM_REG(dev, tracer_registers)) 566 async_event_mask |= (1ull << MLX5_EVENT_TYPE_DEVICE_TRACER); 567 568 if (MLX5_CAP_GEN(dev, max_num_of_monitor_counters)) 569 async_event_mask |= (1ull << MLX5_EVENT_TYPE_MONITOR_COUNTER); 570 571 if (mlx5_eswitch_is_funcs_handler(dev)) 572 async_event_mask |= 573 (1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED); 574 575 if (MLX5_CAP_GEN_MAX(dev, vhca_state)) 576 async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE); 577 578 if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload)) 579 async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); 580 581 if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) 582 async_event_mask |= 583 (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); 584 585 mask[0] = async_event_mask; 586 587 if (MLX5_CAP_GEN(dev, event_cap)) 588 gather_user_async_events(dev, mask); 589 } 590 591 static int 592 setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq, 593 struct mlx5_eq_param *param, const char *name) 594 { 595 int err; 596 597 eq->irq_nb.notifier_call = mlx5_eq_async_int; 598 spin_lock_init(&eq->lock); 599 600 err = create_async_eq(dev, &eq->core, param); 601 if (err) { 602 mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err); 603 return err; 604 } 605 err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb); 606 if (err) { 607 mlx5_core_warn(dev, "failed to enable %s EQ %d\n", name, err); 608 destroy_async_eq(dev, &eq->core); 609 } 610 return err; 611 } 612 613 static void cleanup_async_eq(struct mlx5_core_dev *dev, 614 struct mlx5_eq_async *eq, const char *name) 615 { 616 int err; 617 618 mlx5_eq_disable(dev, &eq->core, &eq->irq_nb); 619 err = destroy_async_eq(dev, &eq->core); 620 if (err) 621 mlx5_core_err(dev, "failed to destroy %s eq, err(%d)\n", 622 name, err); 623 } 624 625 static u16 async_eq_depth_devlink_param_get(struct mlx5_core_dev *dev) 626 { 627 struct devlink *devlink = priv_to_devlink(dev); 628 union devlink_param_value val; 629 int err; 630 631 err = devl_param_driverinit_value_get(devlink, 632 DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE, 633 &val); 634 if (!err) 635 return val.vu32; 636 mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err); 637 return MLX5_NUM_ASYNC_EQE; 638 } 639 640 static int create_async_eqs(struct mlx5_core_dev *dev) 641 { 642 struct mlx5_eq_table *table = dev->priv.eq_table; 643 struct mlx5_eq_param param = {}; 644 int err; 645 646 /* All the async_eqs are using single IRQ, request one IRQ and share its 647 * index among all the async_eqs of this device. 648 */ 649 table->ctrl_irq = mlx5_ctrl_irq_request(dev); 650 if (IS_ERR(table->ctrl_irq)) 651 return PTR_ERR(table->ctrl_irq); 652 653 MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR); 654 mlx5_eq_notifier_register(dev, &table->cq_err_nb); 655 656 param = (struct mlx5_eq_param) { 657 .irq = table->ctrl_irq, 658 .nent = MLX5_NUM_CMD_EQE, 659 .mask[0] = 1ull << MLX5_EVENT_TYPE_CMD, 660 }; 661 mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_CREATE_EQ); 662 err = setup_async_eq(dev, &table->cmd_eq, ¶m, "cmd"); 663 if (err) 664 goto err1; 665 666 mlx5_cmd_use_events(dev); 667 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); 668 669 param = (struct mlx5_eq_param) { 670 .irq = table->ctrl_irq, 671 .nent = async_eq_depth_devlink_param_get(dev), 672 }; 673 674 gather_async_events_mask(dev, param.mask); 675 err = setup_async_eq(dev, &table->async_eq, ¶m, "async"); 676 if (err) 677 goto err2; 678 679 param = (struct mlx5_eq_param) { 680 .irq = table->ctrl_irq, 681 .nent = /* TODO: sriov max_vf + */ 1, 682 .mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST, 683 }; 684 685 err = setup_async_eq(dev, &table->pages_eq, ¶m, "pages"); 686 if (err) 687 goto err3; 688 689 return 0; 690 691 err3: 692 cleanup_async_eq(dev, &table->async_eq, "async"); 693 err2: 694 mlx5_cmd_use_polling(dev); 695 cleanup_async_eq(dev, &table->cmd_eq, "cmd"); 696 err1: 697 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); 698 mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); 699 mlx5_ctrl_irq_release(table->ctrl_irq); 700 return err; 701 } 702 703 static void destroy_async_eqs(struct mlx5_core_dev *dev) 704 { 705 struct mlx5_eq_table *table = dev->priv.eq_table; 706 707 cleanup_async_eq(dev, &table->pages_eq, "pages"); 708 cleanup_async_eq(dev, &table->async_eq, "async"); 709 mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_DESTROY_EQ); 710 mlx5_cmd_use_polling(dev); 711 cleanup_async_eq(dev, &table->cmd_eq, "cmd"); 712 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); 713 mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); 714 mlx5_ctrl_irq_release(table->ctrl_irq); 715 } 716 717 struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev) 718 { 719 return &dev->priv.eq_table->async_eq.core; 720 } 721 722 void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev) 723 { 724 synchronize_irq(dev->priv.eq_table->async_eq.core.irqn); 725 } 726 727 void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev) 728 { 729 synchronize_irq(dev->priv.eq_table->cmd_eq.core.irqn); 730 } 731 732 /* Generic EQ API for mlx5_core consumers 733 * Needed For RDMA ODP EQ for now 734 */ 735 struct mlx5_eq * 736 mlx5_eq_create_generic(struct mlx5_core_dev *dev, 737 struct mlx5_eq_param *param) 738 { 739 struct mlx5_eq *eq = kvzalloc_node(sizeof(*eq), GFP_KERNEL, 740 dev->priv.numa_node); 741 int err; 742 743 if (!eq) 744 return ERR_PTR(-ENOMEM); 745 746 param->irq = dev->priv.eq_table->ctrl_irq; 747 err = create_async_eq(dev, eq, param); 748 if (err) { 749 kvfree(eq); 750 eq = ERR_PTR(err); 751 } 752 753 return eq; 754 } 755 EXPORT_SYMBOL(mlx5_eq_create_generic); 756 757 int mlx5_eq_destroy_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq) 758 { 759 int err; 760 761 if (IS_ERR(eq)) 762 return -EINVAL; 763 764 err = destroy_async_eq(dev, eq); 765 if (err) 766 goto out; 767 768 kvfree(eq); 769 out: 770 return err; 771 } 772 EXPORT_SYMBOL(mlx5_eq_destroy_generic); 773 774 struct mlx5_eqe *mlx5_eq_get_eqe(struct mlx5_eq *eq, u32 cc) 775 { 776 u32 ci = eq->cons_index + cc; 777 u32 nent = eq_get_size(eq); 778 struct mlx5_eqe *eqe; 779 780 eqe = get_eqe(eq, ci & (nent - 1)); 781 eqe = ((eqe->owner & 1) ^ !!(ci & nent)) ? NULL : eqe; 782 /* Make sure we read EQ entry contents after we've 783 * checked the ownership bit. 784 */ 785 if (eqe) 786 dma_rmb(); 787 788 return eqe; 789 } 790 EXPORT_SYMBOL(mlx5_eq_get_eqe); 791 792 void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm) 793 { 794 __be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2); 795 u32 val; 796 797 eq->cons_index += cc; 798 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24); 799 800 __raw_writel((__force u32)cpu_to_be32(val), addr); 801 /* We still want ordering, just not swabbing, so add a barrier */ 802 wmb(); 803 } 804 EXPORT_SYMBOL(mlx5_eq_update_ci); 805 806 static void comp_irqs_release_pci(struct mlx5_core_dev *dev) 807 { 808 struct mlx5_eq_table *table = dev->priv.eq_table; 809 810 mlx5_irqs_release_vectors(table->comp_irqs, table->num_comp_eqs); 811 } 812 813 static int comp_irqs_request_pci(struct mlx5_core_dev *dev) 814 { 815 struct mlx5_eq_table *table = dev->priv.eq_table; 816 const struct cpumask *prev = cpu_none_mask; 817 const struct cpumask *mask; 818 int ncomp_eqs; 819 u16 *cpus; 820 int ret; 821 int cpu; 822 int i; 823 824 ncomp_eqs = table->num_comp_eqs; 825 cpus = kcalloc(ncomp_eqs, sizeof(*cpus), GFP_KERNEL); 826 if (!cpus) 827 ret = -ENOMEM; 828 829 i = 0; 830 rcu_read_lock(); 831 for_each_numa_hop_mask(mask, dev->priv.numa_node) { 832 for_each_cpu_andnot(cpu, mask, prev) { 833 cpus[i] = cpu; 834 if (++i == ncomp_eqs) 835 goto spread_done; 836 } 837 prev = mask; 838 } 839 spread_done: 840 rcu_read_unlock(); 841 ret = mlx5_irqs_request_vectors(dev, cpus, ncomp_eqs, table->comp_irqs, &table->rmap); 842 kfree(cpus); 843 return ret; 844 } 845 846 static void comp_irqs_release_sf(struct mlx5_core_dev *dev) 847 { 848 struct mlx5_eq_table *table = dev->priv.eq_table; 849 850 mlx5_irq_affinity_irqs_release(dev, table->comp_irqs, table->num_comp_eqs); 851 } 852 853 static int comp_irqs_request_sf(struct mlx5_core_dev *dev) 854 { 855 struct mlx5_eq_table *table = dev->priv.eq_table; 856 int ncomp_eqs = table->num_comp_eqs; 857 858 return mlx5_irq_affinity_irqs_request_auto(dev, ncomp_eqs, table->comp_irqs); 859 } 860 861 static void comp_irqs_release(struct mlx5_core_dev *dev) 862 { 863 struct mlx5_eq_table *table = dev->priv.eq_table; 864 865 mlx5_core_is_sf(dev) ? comp_irqs_release_sf(dev) : 866 comp_irqs_release_pci(dev); 867 868 kfree(table->comp_irqs); 869 } 870 871 static int comp_irqs_request(struct mlx5_core_dev *dev) 872 { 873 struct mlx5_eq_table *table = dev->priv.eq_table; 874 int ncomp_eqs; 875 int ret; 876 877 ncomp_eqs = table->num_comp_eqs; 878 table->comp_irqs = kcalloc(ncomp_eqs, sizeof(*table->comp_irqs), GFP_KERNEL); 879 if (!table->comp_irqs) 880 return -ENOMEM; 881 882 ret = mlx5_core_is_sf(dev) ? comp_irqs_request_sf(dev) : 883 comp_irqs_request_pci(dev); 884 if (ret < 0) 885 kfree(table->comp_irqs); 886 887 return ret; 888 } 889 890 #ifdef CONFIG_RFS_ACCEL 891 static int alloc_rmap(struct mlx5_core_dev *mdev) 892 { 893 struct mlx5_eq_table *eq_table = mdev->priv.eq_table; 894 895 /* rmap is a mapping between irq number and queue number. 896 * Each irq can be assigned only to a single rmap. 897 * Since SFs share IRQs, rmap mapping cannot function correctly 898 * for irqs that are shared between different core/netdev RX rings. 899 * Hence we don't allow netdev rmap for SFs. 900 */ 901 if (mlx5_core_is_sf(mdev)) 902 return 0; 903 904 eq_table->rmap = alloc_irq_cpu_rmap(eq_table->num_comp_eqs); 905 if (!eq_table->rmap) 906 return -ENOMEM; 907 return 0; 908 } 909 910 static void free_rmap(struct mlx5_core_dev *mdev) 911 { 912 struct mlx5_eq_table *eq_table = mdev->priv.eq_table; 913 914 if (eq_table->rmap) { 915 free_irq_cpu_rmap(eq_table->rmap); 916 eq_table->rmap = NULL; 917 } 918 } 919 #else 920 static int alloc_rmap(struct mlx5_core_dev *mdev) { return 0; } 921 static void free_rmap(struct mlx5_core_dev *mdev) {} 922 #endif 923 924 static void destroy_comp_eqs(struct mlx5_core_dev *dev) 925 { 926 struct mlx5_eq_table *table = dev->priv.eq_table; 927 struct mlx5_eq_comp *eq, *n; 928 929 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 930 list_del(&eq->list); 931 mlx5_eq_disable(dev, &eq->core, &eq->irq_nb); 932 if (destroy_unmap_eq(dev, &eq->core)) 933 mlx5_core_warn(dev, "failed to destroy comp EQ 0x%x\n", 934 eq->core.eqn); 935 tasklet_disable(&eq->tasklet_ctx.task); 936 kfree(eq); 937 } 938 comp_irqs_release(dev); 939 free_rmap(dev); 940 } 941 942 static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev) 943 { 944 struct devlink *devlink = priv_to_devlink(dev); 945 union devlink_param_value val; 946 int err; 947 948 err = devl_param_driverinit_value_get(devlink, 949 DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE, 950 &val); 951 if (!err) 952 return val.vu32; 953 mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err); 954 return MLX5_COMP_EQ_SIZE; 955 } 956 957 static int create_comp_eqs(struct mlx5_core_dev *dev) 958 { 959 struct mlx5_eq_table *table = dev->priv.eq_table; 960 struct mlx5_eq_comp *eq; 961 int ncomp_eqs; 962 int nent; 963 int err; 964 int i; 965 966 err = alloc_rmap(dev); 967 if (err) 968 return err; 969 970 ncomp_eqs = comp_irqs_request(dev); 971 if (ncomp_eqs < 0) { 972 err = ncomp_eqs; 973 goto err_irqs_req; 974 } 975 976 INIT_LIST_HEAD(&table->comp_eqs_list); 977 nent = comp_eq_depth_devlink_param_get(dev); 978 979 for (i = 0; i < ncomp_eqs; i++) { 980 struct mlx5_eq_param param = {}; 981 982 eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, dev->priv.numa_node); 983 if (!eq) { 984 err = -ENOMEM; 985 goto clean; 986 } 987 988 INIT_LIST_HEAD(&eq->tasklet_ctx.list); 989 INIT_LIST_HEAD(&eq->tasklet_ctx.process_list); 990 spin_lock_init(&eq->tasklet_ctx.lock); 991 tasklet_setup(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb); 992 993 eq->irq_nb.notifier_call = mlx5_eq_comp_int; 994 param = (struct mlx5_eq_param) { 995 .irq = table->comp_irqs[i], 996 .nent = nent, 997 }; 998 999 err = create_map_eq(dev, &eq->core, ¶m); 1000 if (err) 1001 goto clean_eq; 1002 err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb); 1003 if (err) { 1004 destroy_unmap_eq(dev, &eq->core); 1005 goto clean_eq; 1006 } 1007 1008 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn); 1009 /* add tail, to keep the list ordered, for mlx5_vector2eqn to work */ 1010 list_add_tail(&eq->list, &table->comp_eqs_list); 1011 } 1012 1013 table->num_comp_eqs = ncomp_eqs; 1014 return 0; 1015 1016 clean_eq: 1017 kfree(eq); 1018 clean: 1019 destroy_comp_eqs(dev); 1020 err_irqs_req: 1021 free_rmap(dev); 1022 return err; 1023 } 1024 1025 static int vector2eqnirqn(struct mlx5_core_dev *dev, int vector, int *eqn, 1026 unsigned int *irqn) 1027 { 1028 struct mlx5_eq_table *table = dev->priv.eq_table; 1029 struct mlx5_eq_comp *eq; 1030 int err = -ENOENT; 1031 int i = 0; 1032 1033 list_for_each_entry(eq, &table->comp_eqs_list, list) { 1034 if (i++ == vector) { 1035 if (irqn) 1036 *irqn = eq->core.irqn; 1037 if (eqn) 1038 *eqn = eq->core.eqn; 1039 err = 0; 1040 break; 1041 } 1042 } 1043 1044 return err; 1045 } 1046 1047 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn) 1048 { 1049 return vector2eqnirqn(dev, vector, eqn, NULL); 1050 } 1051 EXPORT_SYMBOL(mlx5_vector2eqn); 1052 1053 int mlx5_vector2irqn(struct mlx5_core_dev *dev, int vector, unsigned int *irqn) 1054 { 1055 return vector2eqnirqn(dev, vector, NULL, irqn); 1056 } 1057 1058 unsigned int mlx5_comp_vectors_count(struct mlx5_core_dev *dev) 1059 { 1060 return dev->priv.eq_table->num_comp_eqs; 1061 } 1062 EXPORT_SYMBOL(mlx5_comp_vectors_count); 1063 1064 struct cpumask * 1065 mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector) 1066 { 1067 struct mlx5_eq_table *table = dev->priv.eq_table; 1068 struct mlx5_eq_comp *eq; 1069 int i = 0; 1070 1071 list_for_each_entry(eq, &table->comp_eqs_list, list) { 1072 if (i++ == vector) 1073 return mlx5_irq_get_affinity_mask(eq->core.irq); 1074 } 1075 1076 WARN_ON_ONCE(1); 1077 return NULL; 1078 } 1079 EXPORT_SYMBOL(mlx5_comp_irq_get_affinity_mask); 1080 1081 #ifdef CONFIG_RFS_ACCEL 1082 struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev) 1083 { 1084 return dev->priv.eq_table->rmap; 1085 } 1086 #endif 1087 1088 struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn) 1089 { 1090 struct mlx5_eq_table *table = dev->priv.eq_table; 1091 struct mlx5_eq_comp *eq; 1092 1093 list_for_each_entry(eq, &table->comp_eqs_list, list) { 1094 if (eq->core.eqn == eqn) 1095 return eq; 1096 } 1097 1098 return ERR_PTR(-ENOENT); 1099 } 1100 1101 /* This function should only be called after mlx5_cmd_force_teardown_hca */ 1102 void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev) 1103 { 1104 struct mlx5_eq_table *table = dev->priv.eq_table; 1105 1106 mutex_lock(&table->lock); /* sync with create/destroy_async_eq */ 1107 mlx5_irq_table_destroy(dev); 1108 mutex_unlock(&table->lock); 1109 } 1110 1111 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 1112 #define MLX5_MAX_ASYNC_EQS 4 1113 #else 1114 #define MLX5_MAX_ASYNC_EQS 3 1115 #endif 1116 1117 static int get_num_eqs(struct mlx5_core_dev *dev) 1118 { 1119 struct mlx5_eq_table *eq_table = dev->priv.eq_table; 1120 int max_dev_eqs; 1121 int max_eqs_sf; 1122 int num_eqs; 1123 1124 /* If ethernet is disabled we use just a single completion vector to 1125 * have the other vectors available for other drivers using mlx5_core. For 1126 * example, mlx5_vdpa 1127 */ 1128 if (!mlx5_core_is_eth_enabled(dev) && mlx5_eth_supported(dev)) 1129 return 1; 1130 1131 max_dev_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ? 1132 MLX5_CAP_GEN(dev, max_num_eqs) : 1133 1 << MLX5_CAP_GEN(dev, log_max_eq); 1134 1135 num_eqs = min_t(int, mlx5_irq_table_get_num_comp(eq_table->irq_table), 1136 max_dev_eqs - MLX5_MAX_ASYNC_EQS); 1137 if (mlx5_core_is_sf(dev)) { 1138 max_eqs_sf = min_t(int, MLX5_COMP_EQS_PER_SF, 1139 mlx5_irq_table_get_sfs_vec(eq_table->irq_table)); 1140 num_eqs = min_t(int, num_eqs, max_eqs_sf); 1141 } 1142 1143 return num_eqs; 1144 } 1145 1146 int mlx5_eq_table_create(struct mlx5_core_dev *dev) 1147 { 1148 struct mlx5_eq_table *eq_table = dev->priv.eq_table; 1149 int err; 1150 1151 eq_table->num_comp_eqs = get_num_eqs(dev); 1152 err = create_async_eqs(dev); 1153 if (err) { 1154 mlx5_core_err(dev, "Failed to create async EQs\n"); 1155 goto err_async_eqs; 1156 } 1157 1158 err = create_comp_eqs(dev); 1159 if (err) { 1160 mlx5_core_err(dev, "Failed to create completion EQs\n"); 1161 goto err_comp_eqs; 1162 } 1163 1164 return 0; 1165 1166 err_comp_eqs: 1167 destroy_async_eqs(dev); 1168 err_async_eqs: 1169 return err; 1170 } 1171 1172 void mlx5_eq_table_destroy(struct mlx5_core_dev *dev) 1173 { 1174 destroy_comp_eqs(dev); 1175 destroy_async_eqs(dev); 1176 } 1177 1178 int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb) 1179 { 1180 struct mlx5_eq_table *eqt = dev->priv.eq_table; 1181 1182 return atomic_notifier_chain_register(&eqt->nh[nb->event_type], &nb->nb); 1183 } 1184 EXPORT_SYMBOL(mlx5_eq_notifier_register); 1185 1186 int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb) 1187 { 1188 struct mlx5_eq_table *eqt = dev->priv.eq_table; 1189 1190 return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb); 1191 } 1192 EXPORT_SYMBOL(mlx5_eq_notifier_unregister); 1193