1 /* 2 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/init.h> 35 #include <linux/interrupt.h> 36 #include <linux/slab.h> 37 #include <linux/export.h> 38 #include <linux/mm.h> 39 #include <linux/dma-mapping.h> 40 41 #include <linux/mlx4/cmd.h> 42 #include <linux/cpu_rmap.h> 43 44 #include "mlx4.h" 45 #include "fw.h" 46 47 enum { 48 MLX4_IRQNAME_SIZE = 32 49 }; 50 51 enum { 52 MLX4_NUM_ASYNC_EQE = 0x100, 53 MLX4_NUM_SPARE_EQE = 0x80, 54 MLX4_EQ_ENTRY_SIZE = 0x20 55 }; 56 57 #define MLX4_EQ_STATUS_OK ( 0 << 28) 58 #define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28) 59 #define MLX4_EQ_OWNER_SW ( 0 << 24) 60 #define MLX4_EQ_OWNER_HW ( 1 << 24) 61 #define MLX4_EQ_FLAG_EC ( 1 << 18) 62 #define MLX4_EQ_FLAG_OI ( 1 << 17) 63 #define MLX4_EQ_STATE_ARMED ( 9 << 8) 64 #define MLX4_EQ_STATE_FIRED (10 << 8) 65 #define MLX4_EQ_STATE_ALWAYS_ARMED (11 << 8) 66 67 #define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG) | \ 68 (1ull << MLX4_EVENT_TYPE_COMM_EST) | \ 69 (1ull << MLX4_EVENT_TYPE_SQ_DRAINED) | \ 70 (1ull << MLX4_EVENT_TYPE_CQ_ERROR) | \ 71 (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR) | \ 72 (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR) | \ 73 (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED) | \ 74 (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ 75 (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR) | \ 76 (1ull << MLX4_EVENT_TYPE_PORT_CHANGE) | \ 77 (1ull << MLX4_EVENT_TYPE_ECC_DETECT) | \ 78 (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR) | \ 79 (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE) | \ 80 (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT) | \ 81 (1ull << MLX4_EVENT_TYPE_CMD) | \ 82 (1ull << MLX4_EVENT_TYPE_COMM_CHANNEL) | \ 83 (1ull << MLX4_EVENT_TYPE_FLR_EVENT) | \ 84 (1ull << MLX4_EVENT_TYPE_FATAL_WARNING)) 85 86 static u64 get_async_ev_mask(struct mlx4_dev *dev) 87 { 88 u64 async_ev_mask = MLX4_ASYNC_EVENT_MASK; 89 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV) 90 async_ev_mask |= (1ull << MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT); 91 92 return async_ev_mask; 93 } 94 95 static void eq_set_ci(struct mlx4_eq *eq, int req_not) 96 { 97 __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) | 98 req_not << 31), 99 eq->doorbell); 100 /* We still want ordering, just not swabbing, so add a barrier */ 101 mb(); 102 } 103 104 static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry) 105 { 106 unsigned long off = (entry & (eq->nent - 1)) * MLX4_EQ_ENTRY_SIZE; 107 return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE; 108 } 109 110 static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq) 111 { 112 struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index); 113 return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe; 114 } 115 116 static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_eq) 117 { 118 struct mlx4_eqe *eqe = 119 &slave_eq->event_eqe[slave_eq->cons & (SLAVE_EVENT_EQ_SIZE - 1)]; 120 return (!!(eqe->owner & 0x80) ^ 121 !!(slave_eq->cons & SLAVE_EVENT_EQ_SIZE)) ? 122 eqe : NULL; 123 } 124 125 void mlx4_gen_slave_eqe(struct work_struct *work) 126 { 127 struct mlx4_mfunc_master_ctx *master = 128 container_of(work, struct mlx4_mfunc_master_ctx, 129 slave_event_work); 130 struct mlx4_mfunc *mfunc = 131 container_of(master, struct mlx4_mfunc, master); 132 struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc); 133 struct mlx4_dev *dev = &priv->dev; 134 struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq; 135 struct mlx4_eqe *eqe; 136 u8 slave; 137 int i; 138 139 for (eqe = next_slave_event_eqe(slave_eq); eqe; 140 eqe = next_slave_event_eqe(slave_eq)) { 141 slave = eqe->slave_id; 142 143 /* All active slaves need to receive the event */ 144 if (slave == ALL_SLAVES) { 145 for (i = 0; i < dev->num_slaves; i++) { 146 if (i != dev->caps.function && 147 master->slave_state[i].active) 148 if (mlx4_GEN_EQE(dev, i, eqe)) 149 mlx4_warn(dev, "Failed to " 150 " generate event " 151 "for slave %d\n", i); 152 } 153 } else { 154 if (mlx4_GEN_EQE(dev, slave, eqe)) 155 mlx4_warn(dev, "Failed to generate event " 156 "for slave %d\n", slave); 157 } 158 ++slave_eq->cons; 159 } 160 } 161 162 163 static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe) 164 { 165 struct mlx4_priv *priv = mlx4_priv(dev); 166 struct mlx4_slave_event_eq *slave_eq = &priv->mfunc.master.slave_eq; 167 struct mlx4_eqe *s_eqe = 168 &slave_eq->event_eqe[slave_eq->prod & (SLAVE_EVENT_EQ_SIZE - 1)]; 169 170 if ((!!(s_eqe->owner & 0x80)) ^ 171 (!!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE))) { 172 mlx4_warn(dev, "Master failed to generate an EQE for slave: %d. " 173 "No free EQE on slave events queue\n", slave); 174 return; 175 } 176 177 memcpy(s_eqe, eqe, sizeof(struct mlx4_eqe) - 1); 178 s_eqe->slave_id = slave; 179 /* ensure all information is written before setting the ownersip bit */ 180 wmb(); 181 s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80; 182 ++slave_eq->prod; 183 184 queue_work(priv->mfunc.master.comm_wq, 185 &priv->mfunc.master.slave_event_work); 186 } 187 188 static void mlx4_slave_event(struct mlx4_dev *dev, int slave, 189 struct mlx4_eqe *eqe) 190 { 191 struct mlx4_priv *priv = mlx4_priv(dev); 192 struct mlx4_slave_state *s_slave = 193 &priv->mfunc.master.slave_state[slave]; 194 195 if (!s_slave->active) { 196 /*mlx4_warn(dev, "Trying to pass event to inactive slave\n");*/ 197 return; 198 } 199 200 slave_event(dev, slave, eqe); 201 } 202 203 void mlx4_master_handle_slave_flr(struct work_struct *work) 204 { 205 struct mlx4_mfunc_master_ctx *master = 206 container_of(work, struct mlx4_mfunc_master_ctx, 207 slave_flr_event_work); 208 struct mlx4_mfunc *mfunc = 209 container_of(master, struct mlx4_mfunc, master); 210 struct mlx4_priv *priv = 211 container_of(mfunc, struct mlx4_priv, mfunc); 212 struct mlx4_dev *dev = &priv->dev; 213 struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; 214 int i; 215 int err; 216 217 mlx4_dbg(dev, "mlx4_handle_slave_flr\n"); 218 219 for (i = 0 ; i < dev->num_slaves; i++) { 220 221 if (MLX4_COMM_CMD_FLR == slave_state[i].last_cmd) { 222 mlx4_dbg(dev, "mlx4_handle_slave_flr: " 223 "clean slave: %d\n", i); 224 225 mlx4_delete_all_resources_for_slave(dev, i); 226 /*return the slave to running mode*/ 227 spin_lock(&priv->mfunc.master.slave_state_lock); 228 slave_state[i].last_cmd = MLX4_COMM_CMD_RESET; 229 slave_state[i].is_slave_going_down = 0; 230 spin_unlock(&priv->mfunc.master.slave_state_lock); 231 /*notify the FW:*/ 232 err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE, 233 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 234 if (err) 235 mlx4_warn(dev, "Failed to notify FW on " 236 "FLR done (slave:%d)\n", i); 237 } 238 } 239 } 240 241 static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq) 242 { 243 struct mlx4_priv *priv = mlx4_priv(dev); 244 struct mlx4_eqe *eqe; 245 int cqn; 246 int eqes_found = 0; 247 int set_ci = 0; 248 int port; 249 int slave = 0; 250 int ret; 251 u32 flr_slave; 252 u8 update_slave_state; 253 int i; 254 255 while ((eqe = next_eqe_sw(eq))) { 256 /* 257 * Make sure we read EQ entry contents after we've 258 * checked the ownership bit. 259 */ 260 rmb(); 261 262 switch (eqe->type) { 263 case MLX4_EVENT_TYPE_COMP: 264 cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff; 265 mlx4_cq_completion(dev, cqn); 266 break; 267 268 case MLX4_EVENT_TYPE_PATH_MIG: 269 case MLX4_EVENT_TYPE_COMM_EST: 270 case MLX4_EVENT_TYPE_SQ_DRAINED: 271 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: 272 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: 273 case MLX4_EVENT_TYPE_PATH_MIG_FAILED: 274 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: 275 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: 276 mlx4_dbg(dev, "event %d arrived\n", eqe->type); 277 if (mlx4_is_master(dev)) { 278 /* forward only to slave owning the QP */ 279 ret = mlx4_get_slave_from_resource_id(dev, 280 RES_QP, 281 be32_to_cpu(eqe->event.qp.qpn) 282 & 0xffffff, &slave); 283 if (ret && ret != -ENOENT) { 284 mlx4_dbg(dev, "QP event %02x(%02x) on " 285 "EQ %d at index %u: could " 286 "not get slave id (%d)\n", 287 eqe->type, eqe->subtype, 288 eq->eqn, eq->cons_index, ret); 289 break; 290 } 291 292 if (!ret && slave != dev->caps.function) { 293 mlx4_slave_event(dev, slave, eqe); 294 break; 295 } 296 297 } 298 mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 299 0xffffff, eqe->type); 300 break; 301 302 case MLX4_EVENT_TYPE_SRQ_LIMIT: 303 mlx4_warn(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT\n", 304 __func__); 305 case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: 306 if (mlx4_is_master(dev)) { 307 /* forward only to slave owning the SRQ */ 308 ret = mlx4_get_slave_from_resource_id(dev, 309 RES_SRQ, 310 be32_to_cpu(eqe->event.srq.srqn) 311 & 0xffffff, 312 &slave); 313 if (ret && ret != -ENOENT) { 314 mlx4_warn(dev, "SRQ event %02x(%02x) " 315 "on EQ %d at index %u: could" 316 " not get slave id (%d)\n", 317 eqe->type, eqe->subtype, 318 eq->eqn, eq->cons_index, ret); 319 break; 320 } 321 mlx4_warn(dev, "%s: slave:%d, srq_no:0x%x," 322 " event: %02x(%02x)\n", __func__, 323 slave, 324 be32_to_cpu(eqe->event.srq.srqn), 325 eqe->type, eqe->subtype); 326 327 if (!ret && slave != dev->caps.function) { 328 mlx4_warn(dev, "%s: sending event " 329 "%02x(%02x) to slave:%d\n", 330 __func__, eqe->type, 331 eqe->subtype, slave); 332 mlx4_slave_event(dev, slave, eqe); 333 break; 334 } 335 } 336 mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & 337 0xffffff, eqe->type); 338 break; 339 340 case MLX4_EVENT_TYPE_CMD: 341 mlx4_cmd_event(dev, 342 be16_to_cpu(eqe->event.cmd.token), 343 eqe->event.cmd.status, 344 be64_to_cpu(eqe->event.cmd.out_param)); 345 break; 346 347 case MLX4_EVENT_TYPE_PORT_CHANGE: 348 port = be32_to_cpu(eqe->event.port_change.port) >> 28; 349 if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) { 350 mlx4_dispatch_event(dev, 351 MLX4_DEV_EVENT_PORT_DOWN, 352 port); 353 mlx4_priv(dev)->sense.do_sense_port[port] = 1; 354 if (mlx4_is_master(dev)) 355 /*change the state of all slave's port 356 * to down:*/ 357 for (i = 0; i < dev->num_slaves; i++) { 358 mlx4_dbg(dev, "%s: Sending " 359 "MLX4_PORT_CHANGE_SUBTYPE_DOWN" 360 " to slave: %d, port:%d\n", 361 __func__, i, port); 362 if (i == dev->caps.function) 363 continue; 364 mlx4_slave_event(dev, i, eqe); 365 } 366 } else { 367 mlx4_dispatch_event(dev, 368 MLX4_DEV_EVENT_PORT_UP, 369 port); 370 mlx4_priv(dev)->sense.do_sense_port[port] = 0; 371 372 if (mlx4_is_master(dev)) { 373 for (i = 0; i < dev->num_slaves; i++) { 374 if (i == dev->caps.function) 375 continue; 376 mlx4_slave_event(dev, i, eqe); 377 } 378 } 379 } 380 break; 381 382 case MLX4_EVENT_TYPE_CQ_ERROR: 383 mlx4_warn(dev, "CQ %s on CQN %06x\n", 384 eqe->event.cq_err.syndrome == 1 ? 385 "overrun" : "access violation", 386 be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff); 387 if (mlx4_is_master(dev)) { 388 ret = mlx4_get_slave_from_resource_id(dev, 389 RES_CQ, 390 be32_to_cpu(eqe->event.cq_err.cqn) 391 & 0xffffff, &slave); 392 if (ret && ret != -ENOENT) { 393 mlx4_dbg(dev, "CQ event %02x(%02x) on " 394 "EQ %d at index %u: could " 395 "not get slave id (%d)\n", 396 eqe->type, eqe->subtype, 397 eq->eqn, eq->cons_index, ret); 398 break; 399 } 400 401 if (!ret && slave != dev->caps.function) { 402 mlx4_slave_event(dev, slave, eqe); 403 break; 404 } 405 } 406 mlx4_cq_event(dev, 407 be32_to_cpu(eqe->event.cq_err.cqn) 408 & 0xffffff, 409 eqe->type); 410 break; 411 412 case MLX4_EVENT_TYPE_EQ_OVERFLOW: 413 mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn); 414 break; 415 416 case MLX4_EVENT_TYPE_COMM_CHANNEL: 417 if (!mlx4_is_master(dev)) { 418 mlx4_warn(dev, "Received comm channel event " 419 "for non master device\n"); 420 break; 421 } 422 memcpy(&priv->mfunc.master.comm_arm_bit_vector, 423 eqe->event.comm_channel_arm.bit_vec, 424 sizeof eqe->event.comm_channel_arm.bit_vec); 425 queue_work(priv->mfunc.master.comm_wq, 426 &priv->mfunc.master.comm_work); 427 break; 428 429 case MLX4_EVENT_TYPE_FLR_EVENT: 430 flr_slave = be32_to_cpu(eqe->event.flr_event.slave_id); 431 if (!mlx4_is_master(dev)) { 432 mlx4_warn(dev, "Non-master function received" 433 "FLR event\n"); 434 break; 435 } 436 437 mlx4_dbg(dev, "FLR event for slave: %d\n", flr_slave); 438 439 if (flr_slave >= dev->num_slaves) { 440 mlx4_warn(dev, 441 "Got FLR for unknown function: %d\n", 442 flr_slave); 443 update_slave_state = 0; 444 } else 445 update_slave_state = 1; 446 447 spin_lock(&priv->mfunc.master.slave_state_lock); 448 if (update_slave_state) { 449 priv->mfunc.master.slave_state[flr_slave].active = false; 450 priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR; 451 priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1; 452 } 453 spin_unlock(&priv->mfunc.master.slave_state_lock); 454 queue_work(priv->mfunc.master.comm_wq, 455 &priv->mfunc.master.slave_flr_event_work); 456 break; 457 458 case MLX4_EVENT_TYPE_FATAL_WARNING: 459 if (eqe->subtype == MLX4_FATAL_WARNING_SUBTYPE_WARMING) { 460 if (mlx4_is_master(dev)) 461 for (i = 0; i < dev->num_slaves; i++) { 462 mlx4_dbg(dev, "%s: Sending " 463 "MLX4_FATAL_WARNING_SUBTYPE_WARMING" 464 " to slave: %d\n", __func__, i); 465 if (i == dev->caps.function) 466 continue; 467 mlx4_slave_event(dev, i, eqe); 468 } 469 mlx4_err(dev, "Temperature Threshold was reached! " 470 "Threshold: %d celsius degrees; " 471 "Current Temperature: %d\n", 472 be16_to_cpu(eqe->event.warming.warning_threshold), 473 be16_to_cpu(eqe->event.warming.current_temperature)); 474 } else 475 mlx4_warn(dev, "Unhandled event FATAL WARNING (%02x), " 476 "subtype %02x on EQ %d at index %u. owner=%x, " 477 "nent=0x%x, slave=%x, ownership=%s\n", 478 eqe->type, eqe->subtype, eq->eqn, 479 eq->cons_index, eqe->owner, eq->nent, 480 eqe->slave_id, 481 !!(eqe->owner & 0x80) ^ 482 !!(eq->cons_index & eq->nent) ? "HW" : "SW"); 483 484 break; 485 486 case MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT: 487 mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_MGMT_CHANGE, 488 (unsigned long) eqe); 489 break; 490 491 case MLX4_EVENT_TYPE_EEC_CATAS_ERROR: 492 case MLX4_EVENT_TYPE_ECC_DETECT: 493 default: 494 mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at " 495 "index %u. owner=%x, nent=0x%x, slave=%x, " 496 "ownership=%s\n", 497 eqe->type, eqe->subtype, eq->eqn, 498 eq->cons_index, eqe->owner, eq->nent, 499 eqe->slave_id, 500 !!(eqe->owner & 0x80) ^ 501 !!(eq->cons_index & eq->nent) ? "HW" : "SW"); 502 break; 503 }; 504 505 ++eq->cons_index; 506 eqes_found = 1; 507 ++set_ci; 508 509 /* 510 * The HCA will think the queue has overflowed if we 511 * don't tell it we've been processing events. We 512 * create our EQs with MLX4_NUM_SPARE_EQE extra 513 * entries, so we must update our consumer index at 514 * least that often. 515 */ 516 if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) { 517 eq_set_ci(eq, 0); 518 set_ci = 0; 519 } 520 } 521 522 eq_set_ci(eq, 1); 523 524 return eqes_found; 525 } 526 527 static irqreturn_t mlx4_interrupt(int irq, void *dev_ptr) 528 { 529 struct mlx4_dev *dev = dev_ptr; 530 struct mlx4_priv *priv = mlx4_priv(dev); 531 int work = 0; 532 int i; 533 534 writel(priv->eq_table.clr_mask, priv->eq_table.clr_int); 535 536 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) 537 work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]); 538 539 return IRQ_RETVAL(work); 540 } 541 542 static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr) 543 { 544 struct mlx4_eq *eq = eq_ptr; 545 struct mlx4_dev *dev = eq->dev; 546 547 mlx4_eq_int(dev, eq); 548 549 /* MSI-X vectors always belong to us */ 550 return IRQ_HANDLED; 551 } 552 553 int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave, 554 struct mlx4_vhcr *vhcr, 555 struct mlx4_cmd_mailbox *inbox, 556 struct mlx4_cmd_mailbox *outbox, 557 struct mlx4_cmd_info *cmd) 558 { 559 struct mlx4_priv *priv = mlx4_priv(dev); 560 struct mlx4_slave_event_eq_info *event_eq = 561 priv->mfunc.master.slave_state[slave].event_eq; 562 u32 in_modifier = vhcr->in_modifier; 563 u32 eqn = in_modifier & 0x1FF; 564 u64 in_param = vhcr->in_param; 565 int err = 0; 566 int i; 567 568 if (slave == dev->caps.function) 569 err = mlx4_cmd(dev, in_param, (in_modifier & 0x80000000) | eqn, 570 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, 571 MLX4_CMD_NATIVE); 572 if (!err) 573 for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) 574 if (in_param & (1LL << i)) 575 event_eq[i].eqn = in_modifier >> 31 ? -1 : eqn; 576 577 return err; 578 } 579 580 static int mlx4_MAP_EQ(struct mlx4_dev *dev, u64 event_mask, int unmap, 581 int eq_num) 582 { 583 return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num, 584 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, 585 MLX4_CMD_WRAPPED); 586 } 587 588 static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 589 int eq_num) 590 { 591 return mlx4_cmd(dev, mailbox->dma, eq_num, 0, 592 MLX4_CMD_SW2HW_EQ, MLX4_CMD_TIME_CLASS_A, 593 MLX4_CMD_WRAPPED); 594 } 595 596 static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 597 int eq_num) 598 { 599 return mlx4_cmd_box(dev, 0, mailbox->dma, eq_num, 600 0, MLX4_CMD_HW2SW_EQ, MLX4_CMD_TIME_CLASS_A, 601 MLX4_CMD_WRAPPED); 602 } 603 604 static int mlx4_num_eq_uar(struct mlx4_dev *dev) 605 { 606 /* 607 * Each UAR holds 4 EQ doorbells. To figure out how many UARs 608 * we need to map, take the difference of highest index and 609 * the lowest index we'll use and add 1. 610 */ 611 return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs + 612 dev->caps.comp_pool)/4 - dev->caps.reserved_eqs/4 + 1; 613 } 614 615 static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq) 616 { 617 struct mlx4_priv *priv = mlx4_priv(dev); 618 int index; 619 620 index = eq->eqn / 4 - dev->caps.reserved_eqs / 4; 621 622 if (!priv->eq_table.uar_map[index]) { 623 priv->eq_table.uar_map[index] = 624 ioremap(pci_resource_start(dev->pdev, 2) + 625 ((eq->eqn / 4) << PAGE_SHIFT), 626 PAGE_SIZE); 627 if (!priv->eq_table.uar_map[index]) { 628 mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n", 629 eq->eqn); 630 return NULL; 631 } 632 } 633 634 return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4); 635 } 636 637 static int mlx4_create_eq(struct mlx4_dev *dev, int nent, 638 u8 intr, struct mlx4_eq *eq) 639 { 640 struct mlx4_priv *priv = mlx4_priv(dev); 641 struct mlx4_cmd_mailbox *mailbox; 642 struct mlx4_eq_context *eq_context; 643 int npages; 644 u64 *dma_list = NULL; 645 dma_addr_t t; 646 u64 mtt_addr; 647 int err = -ENOMEM; 648 int i; 649 650 eq->dev = dev; 651 eq->nent = roundup_pow_of_two(max(nent, 2)); 652 npages = PAGE_ALIGN(eq->nent * MLX4_EQ_ENTRY_SIZE) / PAGE_SIZE; 653 654 eq->page_list = kmalloc(npages * sizeof *eq->page_list, 655 GFP_KERNEL); 656 if (!eq->page_list) 657 goto err_out; 658 659 for (i = 0; i < npages; ++i) 660 eq->page_list[i].buf = NULL; 661 662 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); 663 if (!dma_list) 664 goto err_out_free; 665 666 mailbox = mlx4_alloc_cmd_mailbox(dev); 667 if (IS_ERR(mailbox)) 668 goto err_out_free; 669 eq_context = mailbox->buf; 670 671 for (i = 0; i < npages; ++i) { 672 eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, 673 PAGE_SIZE, &t, GFP_KERNEL); 674 if (!eq->page_list[i].buf) 675 goto err_out_free_pages; 676 677 dma_list[i] = t; 678 eq->page_list[i].map = t; 679 680 memset(eq->page_list[i].buf, 0, PAGE_SIZE); 681 } 682 683 eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap); 684 if (eq->eqn == -1) 685 goto err_out_free_pages; 686 687 eq->doorbell = mlx4_get_eq_uar(dev, eq); 688 if (!eq->doorbell) { 689 err = -ENOMEM; 690 goto err_out_free_eq; 691 } 692 693 err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt); 694 if (err) 695 goto err_out_free_eq; 696 697 err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list); 698 if (err) 699 goto err_out_free_mtt; 700 701 memset(eq_context, 0, sizeof *eq_context); 702 eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK | 703 MLX4_EQ_STATE_ARMED); 704 eq_context->log_eq_size = ilog2(eq->nent); 705 eq_context->intr = intr; 706 eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT; 707 708 mtt_addr = mlx4_mtt_addr(dev, &eq->mtt); 709 eq_context->mtt_base_addr_h = mtt_addr >> 32; 710 eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); 711 712 err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn); 713 if (err) { 714 mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err); 715 goto err_out_free_mtt; 716 } 717 718 kfree(dma_list); 719 mlx4_free_cmd_mailbox(dev, mailbox); 720 721 eq->cons_index = 0; 722 723 return err; 724 725 err_out_free_mtt: 726 mlx4_mtt_cleanup(dev, &eq->mtt); 727 728 err_out_free_eq: 729 mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn); 730 731 err_out_free_pages: 732 for (i = 0; i < npages; ++i) 733 if (eq->page_list[i].buf) 734 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 735 eq->page_list[i].buf, 736 eq->page_list[i].map); 737 738 mlx4_free_cmd_mailbox(dev, mailbox); 739 740 err_out_free: 741 kfree(eq->page_list); 742 kfree(dma_list); 743 744 err_out: 745 return err; 746 } 747 748 static void mlx4_free_eq(struct mlx4_dev *dev, 749 struct mlx4_eq *eq) 750 { 751 struct mlx4_priv *priv = mlx4_priv(dev); 752 struct mlx4_cmd_mailbox *mailbox; 753 int err; 754 int npages = PAGE_ALIGN(MLX4_EQ_ENTRY_SIZE * eq->nent) / PAGE_SIZE; 755 int i; 756 757 mailbox = mlx4_alloc_cmd_mailbox(dev); 758 if (IS_ERR(mailbox)) 759 return; 760 761 err = mlx4_HW2SW_EQ(dev, mailbox, eq->eqn); 762 if (err) 763 mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err); 764 765 if (0) { 766 mlx4_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn); 767 for (i = 0; i < sizeof (struct mlx4_eq_context) / 4; ++i) { 768 if (i % 4 == 0) 769 pr_cont("[%02x] ", i * 4); 770 pr_cont(" %08x", be32_to_cpup(mailbox->buf + i * 4)); 771 if ((i + 1) % 4 == 0) 772 pr_cont("\n"); 773 } 774 } 775 776 mlx4_mtt_cleanup(dev, &eq->mtt); 777 for (i = 0; i < npages; ++i) 778 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 779 eq->page_list[i].buf, 780 eq->page_list[i].map); 781 782 kfree(eq->page_list); 783 mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn); 784 mlx4_free_cmd_mailbox(dev, mailbox); 785 } 786 787 static void mlx4_free_irqs(struct mlx4_dev *dev) 788 { 789 struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table; 790 struct mlx4_priv *priv = mlx4_priv(dev); 791 int i, vec; 792 793 if (eq_table->have_irq) 794 free_irq(dev->pdev->irq, dev); 795 796 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) 797 if (eq_table->eq[i].have_irq) { 798 free_irq(eq_table->eq[i].irq, eq_table->eq + i); 799 eq_table->eq[i].have_irq = 0; 800 } 801 802 for (i = 0; i < dev->caps.comp_pool; i++) { 803 /* 804 * Freeing the assigned irq's 805 * all bits should be 0, but we need to validate 806 */ 807 if (priv->msix_ctl.pool_bm & 1ULL << i) { 808 /* NO need protecting*/ 809 vec = dev->caps.num_comp_vectors + 1 + i; 810 free_irq(priv->eq_table.eq[vec].irq, 811 &priv->eq_table.eq[vec]); 812 } 813 } 814 815 816 kfree(eq_table->irq_names); 817 } 818 819 static int mlx4_map_clr_int(struct mlx4_dev *dev) 820 { 821 struct mlx4_priv *priv = mlx4_priv(dev); 822 823 priv->clr_base = ioremap(pci_resource_start(dev->pdev, priv->fw.clr_int_bar) + 824 priv->fw.clr_int_base, MLX4_CLR_INT_SIZE); 825 if (!priv->clr_base) { 826 mlx4_err(dev, "Couldn't map interrupt clear register, aborting.\n"); 827 return -ENOMEM; 828 } 829 830 return 0; 831 } 832 833 static void mlx4_unmap_clr_int(struct mlx4_dev *dev) 834 { 835 struct mlx4_priv *priv = mlx4_priv(dev); 836 837 iounmap(priv->clr_base); 838 } 839 840 int mlx4_alloc_eq_table(struct mlx4_dev *dev) 841 { 842 struct mlx4_priv *priv = mlx4_priv(dev); 843 844 priv->eq_table.eq = kcalloc(dev->caps.num_eqs - dev->caps.reserved_eqs, 845 sizeof *priv->eq_table.eq, GFP_KERNEL); 846 if (!priv->eq_table.eq) 847 return -ENOMEM; 848 849 return 0; 850 } 851 852 void mlx4_free_eq_table(struct mlx4_dev *dev) 853 { 854 kfree(mlx4_priv(dev)->eq_table.eq); 855 } 856 857 int mlx4_init_eq_table(struct mlx4_dev *dev) 858 { 859 struct mlx4_priv *priv = mlx4_priv(dev); 860 int err; 861 int i; 862 863 priv->eq_table.uar_map = kcalloc(mlx4_num_eq_uar(dev), 864 sizeof *priv->eq_table.uar_map, 865 GFP_KERNEL); 866 if (!priv->eq_table.uar_map) { 867 err = -ENOMEM; 868 goto err_out_free; 869 } 870 871 err = mlx4_bitmap_init(&priv->eq_table.bitmap, dev->caps.num_eqs, 872 dev->caps.num_eqs - 1, dev->caps.reserved_eqs, 0); 873 if (err) 874 goto err_out_free; 875 876 for (i = 0; i < mlx4_num_eq_uar(dev); ++i) 877 priv->eq_table.uar_map[i] = NULL; 878 879 if (!mlx4_is_slave(dev)) { 880 err = mlx4_map_clr_int(dev); 881 if (err) 882 goto err_out_bitmap; 883 884 priv->eq_table.clr_mask = 885 swab32(1 << (priv->eq_table.inta_pin & 31)); 886 priv->eq_table.clr_int = priv->clr_base + 887 (priv->eq_table.inta_pin < 32 ? 4 : 0); 888 } 889 890 priv->eq_table.irq_names = 891 kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1 + 892 dev->caps.comp_pool), 893 GFP_KERNEL); 894 if (!priv->eq_table.irq_names) { 895 err = -ENOMEM; 896 goto err_out_bitmap; 897 } 898 899 for (i = 0; i < dev->caps.num_comp_vectors; ++i) { 900 err = mlx4_create_eq(dev, dev->caps.num_cqs - 901 dev->caps.reserved_cqs + 902 MLX4_NUM_SPARE_EQE, 903 (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, 904 &priv->eq_table.eq[i]); 905 if (err) { 906 --i; 907 goto err_out_unmap; 908 } 909 } 910 911 err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE, 912 (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0, 913 &priv->eq_table.eq[dev->caps.num_comp_vectors]); 914 if (err) 915 goto err_out_comp; 916 917 /*if additional completion vectors poolsize is 0 this loop will not run*/ 918 for (i = dev->caps.num_comp_vectors + 1; 919 i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) { 920 921 err = mlx4_create_eq(dev, dev->caps.num_cqs - 922 dev->caps.reserved_cqs + 923 MLX4_NUM_SPARE_EQE, 924 (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, 925 &priv->eq_table.eq[i]); 926 if (err) { 927 --i; 928 goto err_out_unmap; 929 } 930 } 931 932 933 if (dev->flags & MLX4_FLAG_MSI_X) { 934 const char *eq_name; 935 936 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) { 937 if (i < dev->caps.num_comp_vectors) { 938 snprintf(priv->eq_table.irq_names + 939 i * MLX4_IRQNAME_SIZE, 940 MLX4_IRQNAME_SIZE, 941 "mlx4-comp-%d@pci:%s", i, 942 pci_name(dev->pdev)); 943 } else { 944 snprintf(priv->eq_table.irq_names + 945 i * MLX4_IRQNAME_SIZE, 946 MLX4_IRQNAME_SIZE, 947 "mlx4-async@pci:%s", 948 pci_name(dev->pdev)); 949 } 950 951 eq_name = priv->eq_table.irq_names + 952 i * MLX4_IRQNAME_SIZE; 953 err = request_irq(priv->eq_table.eq[i].irq, 954 mlx4_msi_x_interrupt, 0, eq_name, 955 priv->eq_table.eq + i); 956 if (err) 957 goto err_out_async; 958 959 priv->eq_table.eq[i].have_irq = 1; 960 } 961 } else { 962 snprintf(priv->eq_table.irq_names, 963 MLX4_IRQNAME_SIZE, 964 DRV_NAME "@pci:%s", 965 pci_name(dev->pdev)); 966 err = request_irq(dev->pdev->irq, mlx4_interrupt, 967 IRQF_SHARED, priv->eq_table.irq_names, dev); 968 if (err) 969 goto err_out_async; 970 971 priv->eq_table.have_irq = 1; 972 } 973 974 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, 975 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 976 if (err) 977 mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n", 978 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn, err); 979 980 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) 981 eq_set_ci(&priv->eq_table.eq[i], 1); 982 983 return 0; 984 985 err_out_async: 986 mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]); 987 988 err_out_comp: 989 i = dev->caps.num_comp_vectors - 1; 990 991 err_out_unmap: 992 while (i >= 0) { 993 mlx4_free_eq(dev, &priv->eq_table.eq[i]); 994 --i; 995 } 996 if (!mlx4_is_slave(dev)) 997 mlx4_unmap_clr_int(dev); 998 mlx4_free_irqs(dev); 999 1000 err_out_bitmap: 1001 mlx4_bitmap_cleanup(&priv->eq_table.bitmap); 1002 1003 err_out_free: 1004 kfree(priv->eq_table.uar_map); 1005 1006 return err; 1007 } 1008 1009 void mlx4_cleanup_eq_table(struct mlx4_dev *dev) 1010 { 1011 struct mlx4_priv *priv = mlx4_priv(dev); 1012 int i; 1013 1014 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1, 1015 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 1016 1017 mlx4_free_irqs(dev); 1018 1019 for (i = 0; i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) 1020 mlx4_free_eq(dev, &priv->eq_table.eq[i]); 1021 1022 if (!mlx4_is_slave(dev)) 1023 mlx4_unmap_clr_int(dev); 1024 1025 for (i = 0; i < mlx4_num_eq_uar(dev); ++i) 1026 if (priv->eq_table.uar_map[i]) 1027 iounmap(priv->eq_table.uar_map[i]); 1028 1029 mlx4_bitmap_cleanup(&priv->eq_table.bitmap); 1030 1031 kfree(priv->eq_table.uar_map); 1032 } 1033 1034 /* A test that verifies that we can accept interrupts on all 1035 * the irq vectors of the device. 1036 * Interrupts are checked using the NOP command. 1037 */ 1038 int mlx4_test_interrupts(struct mlx4_dev *dev) 1039 { 1040 struct mlx4_priv *priv = mlx4_priv(dev); 1041 int i; 1042 int err; 1043 1044 err = mlx4_NOP(dev); 1045 /* When not in MSI_X, there is only one irq to check */ 1046 if (!(dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(dev)) 1047 return err; 1048 1049 /* A loop over all completion vectors, for each vector we will check 1050 * whether it works by mapping command completions to that vector 1051 * and performing a NOP command 1052 */ 1053 for(i = 0; !err && (i < dev->caps.num_comp_vectors); ++i) { 1054 /* Temporary use polling for command completions */ 1055 mlx4_cmd_use_polling(dev); 1056 1057 /* Map the new eq to handle all asyncronous events */ 1058 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, 1059 priv->eq_table.eq[i].eqn); 1060 if (err) { 1061 mlx4_warn(dev, "Failed mapping eq for interrupt test\n"); 1062 mlx4_cmd_use_events(dev); 1063 break; 1064 } 1065 1066 /* Go back to using events */ 1067 mlx4_cmd_use_events(dev); 1068 err = mlx4_NOP(dev); 1069 } 1070 1071 /* Return to default */ 1072 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, 1073 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 1074 return err; 1075 } 1076 EXPORT_SYMBOL(mlx4_test_interrupts); 1077 1078 int mlx4_assign_eq(struct mlx4_dev *dev, char *name, struct cpu_rmap *rmap, 1079 int *vector) 1080 { 1081 1082 struct mlx4_priv *priv = mlx4_priv(dev); 1083 int vec = 0, err = 0, i; 1084 1085 mutex_lock(&priv->msix_ctl.pool_lock); 1086 for (i = 0; !vec && i < dev->caps.comp_pool; i++) { 1087 if (~priv->msix_ctl.pool_bm & 1ULL << i) { 1088 priv->msix_ctl.pool_bm |= 1ULL << i; 1089 vec = dev->caps.num_comp_vectors + 1 + i; 1090 snprintf(priv->eq_table.irq_names + 1091 vec * MLX4_IRQNAME_SIZE, 1092 MLX4_IRQNAME_SIZE, "%s", name); 1093 #ifdef CONFIG_RFS_ACCEL 1094 if (rmap) { 1095 err = irq_cpu_rmap_add(rmap, 1096 priv->eq_table.eq[vec].irq); 1097 if (err) 1098 mlx4_warn(dev, "Failed adding irq rmap\n"); 1099 } 1100 #endif 1101 err = request_irq(priv->eq_table.eq[vec].irq, 1102 mlx4_msi_x_interrupt, 0, 1103 &priv->eq_table.irq_names[vec<<5], 1104 priv->eq_table.eq + vec); 1105 if (err) { 1106 /*zero out bit by fliping it*/ 1107 priv->msix_ctl.pool_bm ^= 1 << i; 1108 vec = 0; 1109 continue; 1110 /*we dont want to break here*/ 1111 } 1112 eq_set_ci(&priv->eq_table.eq[vec], 1); 1113 } 1114 } 1115 mutex_unlock(&priv->msix_ctl.pool_lock); 1116 1117 if (vec) { 1118 *vector = vec; 1119 } else { 1120 *vector = 0; 1121 err = (i == dev->caps.comp_pool) ? -ENOSPC : err; 1122 } 1123 return err; 1124 } 1125 EXPORT_SYMBOL(mlx4_assign_eq); 1126 1127 void mlx4_release_eq(struct mlx4_dev *dev, int vec) 1128 { 1129 struct mlx4_priv *priv = mlx4_priv(dev); 1130 /*bm index*/ 1131 int i = vec - dev->caps.num_comp_vectors - 1; 1132 1133 if (likely(i >= 0)) { 1134 /*sanity check , making sure were not trying to free irq's 1135 Belonging to a legacy EQ*/ 1136 mutex_lock(&priv->msix_ctl.pool_lock); 1137 if (priv->msix_ctl.pool_bm & 1ULL << i) { 1138 free_irq(priv->eq_table.eq[vec].irq, 1139 &priv->eq_table.eq[vec]); 1140 priv->msix_ctl.pool_bm &= ~(1ULL << i); 1141 } 1142 mutex_unlock(&priv->msix_ctl.pool_lock); 1143 } 1144 1145 } 1146 EXPORT_SYMBOL(mlx4_release_eq); 1147 1148