1 /* 2 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/init.h> 35 #include <linux/interrupt.h> 36 #include <linux/slab.h> 37 #include <linux/export.h> 38 #include <linux/mm.h> 39 #include <linux/dma-mapping.h> 40 41 #include <linux/mlx4/cmd.h> 42 43 #include "mlx4.h" 44 #include "fw.h" 45 46 enum { 47 MLX4_IRQNAME_SIZE = 32 48 }; 49 50 enum { 51 MLX4_NUM_ASYNC_EQE = 0x100, 52 MLX4_NUM_SPARE_EQE = 0x80, 53 MLX4_EQ_ENTRY_SIZE = 0x20 54 }; 55 56 #define MLX4_EQ_STATUS_OK ( 0 << 28) 57 #define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28) 58 #define MLX4_EQ_OWNER_SW ( 0 << 24) 59 #define MLX4_EQ_OWNER_HW ( 1 << 24) 60 #define MLX4_EQ_FLAG_EC ( 1 << 18) 61 #define MLX4_EQ_FLAG_OI ( 1 << 17) 62 #define MLX4_EQ_STATE_ARMED ( 9 << 8) 63 #define MLX4_EQ_STATE_FIRED (10 << 8) 64 #define MLX4_EQ_STATE_ALWAYS_ARMED (11 << 8) 65 66 #define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG) | \ 67 (1ull << MLX4_EVENT_TYPE_COMM_EST) | \ 68 (1ull << MLX4_EVENT_TYPE_SQ_DRAINED) | \ 69 (1ull << MLX4_EVENT_TYPE_CQ_ERROR) | \ 70 (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR) | \ 71 (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR) | \ 72 (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED) | \ 73 (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ 74 (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR) | \ 75 (1ull << MLX4_EVENT_TYPE_PORT_CHANGE) | \ 76 (1ull << MLX4_EVENT_TYPE_ECC_DETECT) | \ 77 (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR) | \ 78 (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE) | \ 79 (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT) | \ 80 (1ull << MLX4_EVENT_TYPE_CMD) | \ 81 (1ull << MLX4_EVENT_TYPE_COMM_CHANNEL) | \ 82 (1ull << MLX4_EVENT_TYPE_FLR_EVENT) | \ 83 (1ull << MLX4_EVENT_TYPE_FATAL_WARNING)) 84 85 static void eq_set_ci(struct mlx4_eq *eq, int req_not) 86 { 87 __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) | 88 req_not << 31), 89 eq->doorbell); 90 /* We still want ordering, just not swabbing, so add a barrier */ 91 mb(); 92 } 93 94 static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry) 95 { 96 unsigned long off = (entry & (eq->nent - 1)) * MLX4_EQ_ENTRY_SIZE; 97 return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE; 98 } 99 100 static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq) 101 { 102 struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index); 103 return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe; 104 } 105 106 static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_eq) 107 { 108 struct mlx4_eqe *eqe = 109 &slave_eq->event_eqe[slave_eq->cons & (SLAVE_EVENT_EQ_SIZE - 1)]; 110 return (!!(eqe->owner & 0x80) ^ 111 !!(slave_eq->cons & SLAVE_EVENT_EQ_SIZE)) ? 112 eqe : NULL; 113 } 114 115 void mlx4_gen_slave_eqe(struct work_struct *work) 116 { 117 struct mlx4_mfunc_master_ctx *master = 118 container_of(work, struct mlx4_mfunc_master_ctx, 119 slave_event_work); 120 struct mlx4_mfunc *mfunc = 121 container_of(master, struct mlx4_mfunc, master); 122 struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc); 123 struct mlx4_dev *dev = &priv->dev; 124 struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq; 125 struct mlx4_eqe *eqe; 126 u8 slave; 127 int i; 128 129 for (eqe = next_slave_event_eqe(slave_eq); eqe; 130 eqe = next_slave_event_eqe(slave_eq)) { 131 slave = eqe->slave_id; 132 133 /* All active slaves need to receive the event */ 134 if (slave == ALL_SLAVES) { 135 for (i = 0; i < dev->num_slaves; i++) { 136 if (i != dev->caps.function && 137 master->slave_state[i].active) 138 if (mlx4_GEN_EQE(dev, i, eqe)) 139 mlx4_warn(dev, "Failed to " 140 " generate event " 141 "for slave %d\n", i); 142 } 143 } else { 144 if (mlx4_GEN_EQE(dev, slave, eqe)) 145 mlx4_warn(dev, "Failed to generate event " 146 "for slave %d\n", slave); 147 } 148 ++slave_eq->cons; 149 } 150 } 151 152 153 static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe) 154 { 155 struct mlx4_priv *priv = mlx4_priv(dev); 156 struct mlx4_slave_event_eq *slave_eq = &priv->mfunc.master.slave_eq; 157 struct mlx4_eqe *s_eqe = 158 &slave_eq->event_eqe[slave_eq->prod & (SLAVE_EVENT_EQ_SIZE - 1)]; 159 160 if ((!!(s_eqe->owner & 0x80)) ^ 161 (!!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE))) { 162 mlx4_warn(dev, "Master failed to generate an EQE for slave: %d. " 163 "No free EQE on slave events queue\n", slave); 164 return; 165 } 166 167 memcpy(s_eqe, eqe, sizeof(struct mlx4_eqe) - 1); 168 s_eqe->slave_id = slave; 169 /* ensure all information is written before setting the ownersip bit */ 170 wmb(); 171 s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80; 172 ++slave_eq->prod; 173 174 queue_work(priv->mfunc.master.comm_wq, 175 &priv->mfunc.master.slave_event_work); 176 } 177 178 static void mlx4_slave_event(struct mlx4_dev *dev, int slave, 179 struct mlx4_eqe *eqe) 180 { 181 struct mlx4_priv *priv = mlx4_priv(dev); 182 struct mlx4_slave_state *s_slave = 183 &priv->mfunc.master.slave_state[slave]; 184 185 if (!s_slave->active) { 186 /*mlx4_warn(dev, "Trying to pass event to inactive slave\n");*/ 187 return; 188 } 189 190 slave_event(dev, slave, eqe); 191 } 192 193 void mlx4_master_handle_slave_flr(struct work_struct *work) 194 { 195 struct mlx4_mfunc_master_ctx *master = 196 container_of(work, struct mlx4_mfunc_master_ctx, 197 slave_flr_event_work); 198 struct mlx4_mfunc *mfunc = 199 container_of(master, struct mlx4_mfunc, master); 200 struct mlx4_priv *priv = 201 container_of(mfunc, struct mlx4_priv, mfunc); 202 struct mlx4_dev *dev = &priv->dev; 203 struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state; 204 int i; 205 int err; 206 207 mlx4_dbg(dev, "mlx4_handle_slave_flr\n"); 208 209 for (i = 0 ; i < dev->num_slaves; i++) { 210 211 if (MLX4_COMM_CMD_FLR == slave_state[i].last_cmd) { 212 mlx4_dbg(dev, "mlx4_handle_slave_flr: " 213 "clean slave: %d\n", i); 214 215 mlx4_delete_all_resources_for_slave(dev, i); 216 /*return the slave to running mode*/ 217 spin_lock(&priv->mfunc.master.slave_state_lock); 218 slave_state[i].last_cmd = MLX4_COMM_CMD_RESET; 219 slave_state[i].is_slave_going_down = 0; 220 spin_unlock(&priv->mfunc.master.slave_state_lock); 221 /*notify the FW:*/ 222 err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE, 223 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 224 if (err) 225 mlx4_warn(dev, "Failed to notify FW on " 226 "FLR done (slave:%d)\n", i); 227 } 228 } 229 } 230 231 static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq) 232 { 233 struct mlx4_priv *priv = mlx4_priv(dev); 234 struct mlx4_eqe *eqe; 235 int cqn; 236 int eqes_found = 0; 237 int set_ci = 0; 238 int port; 239 int slave = 0; 240 int ret; 241 u32 flr_slave; 242 u8 update_slave_state; 243 int i; 244 245 while ((eqe = next_eqe_sw(eq))) { 246 /* 247 * Make sure we read EQ entry contents after we've 248 * checked the ownership bit. 249 */ 250 rmb(); 251 252 switch (eqe->type) { 253 case MLX4_EVENT_TYPE_COMP: 254 cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff; 255 mlx4_cq_completion(dev, cqn); 256 break; 257 258 case MLX4_EVENT_TYPE_PATH_MIG: 259 case MLX4_EVENT_TYPE_COMM_EST: 260 case MLX4_EVENT_TYPE_SQ_DRAINED: 261 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE: 262 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR: 263 case MLX4_EVENT_TYPE_PATH_MIG_FAILED: 264 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR: 265 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR: 266 mlx4_dbg(dev, "event %d arrived\n", eqe->type); 267 if (mlx4_is_master(dev)) { 268 /* forward only to slave owning the QP */ 269 ret = mlx4_get_slave_from_resource_id(dev, 270 RES_QP, 271 be32_to_cpu(eqe->event.qp.qpn) 272 & 0xffffff, &slave); 273 if (ret && ret != -ENOENT) { 274 mlx4_dbg(dev, "QP event %02x(%02x) on " 275 "EQ %d at index %u: could " 276 "not get slave id (%d)\n", 277 eqe->type, eqe->subtype, 278 eq->eqn, eq->cons_index, ret); 279 break; 280 } 281 282 if (!ret && slave != dev->caps.function) { 283 mlx4_slave_event(dev, slave, eqe); 284 break; 285 } 286 287 } 288 mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 289 0xffffff, eqe->type); 290 break; 291 292 case MLX4_EVENT_TYPE_SRQ_LIMIT: 293 mlx4_warn(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT\n", 294 __func__); 295 case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: 296 if (mlx4_is_master(dev)) { 297 /* forward only to slave owning the SRQ */ 298 ret = mlx4_get_slave_from_resource_id(dev, 299 RES_SRQ, 300 be32_to_cpu(eqe->event.srq.srqn) 301 & 0xffffff, 302 &slave); 303 if (ret && ret != -ENOENT) { 304 mlx4_warn(dev, "SRQ event %02x(%02x) " 305 "on EQ %d at index %u: could" 306 " not get slave id (%d)\n", 307 eqe->type, eqe->subtype, 308 eq->eqn, eq->cons_index, ret); 309 break; 310 } 311 mlx4_warn(dev, "%s: slave:%d, srq_no:0x%x," 312 " event: %02x(%02x)\n", __func__, 313 slave, 314 be32_to_cpu(eqe->event.srq.srqn), 315 eqe->type, eqe->subtype); 316 317 if (!ret && slave != dev->caps.function) { 318 mlx4_warn(dev, "%s: sending event " 319 "%02x(%02x) to slave:%d\n", 320 __func__, eqe->type, 321 eqe->subtype, slave); 322 mlx4_slave_event(dev, slave, eqe); 323 break; 324 } 325 } 326 mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & 327 0xffffff, eqe->type); 328 break; 329 330 case MLX4_EVENT_TYPE_CMD: 331 mlx4_cmd_event(dev, 332 be16_to_cpu(eqe->event.cmd.token), 333 eqe->event.cmd.status, 334 be64_to_cpu(eqe->event.cmd.out_param)); 335 break; 336 337 case MLX4_EVENT_TYPE_PORT_CHANGE: 338 port = be32_to_cpu(eqe->event.port_change.port) >> 28; 339 if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) { 340 mlx4_dispatch_event(dev, 341 MLX4_DEV_EVENT_PORT_DOWN, 342 port); 343 mlx4_priv(dev)->sense.do_sense_port[port] = 1; 344 if (mlx4_is_master(dev)) 345 /*change the state of all slave's port 346 * to down:*/ 347 for (i = 0; i < dev->num_slaves; i++) { 348 mlx4_dbg(dev, "%s: Sending " 349 "MLX4_PORT_CHANGE_SUBTYPE_DOWN" 350 " to slave: %d, port:%d\n", 351 __func__, i, port); 352 if (i == dev->caps.function) 353 continue; 354 mlx4_slave_event(dev, i, eqe); 355 } 356 } else { 357 mlx4_dispatch_event(dev, 358 MLX4_DEV_EVENT_PORT_UP, 359 port); 360 mlx4_priv(dev)->sense.do_sense_port[port] = 0; 361 362 if (mlx4_is_master(dev)) { 363 for (i = 0; i < dev->num_slaves; i++) { 364 if (i == dev->caps.function) 365 continue; 366 mlx4_slave_event(dev, i, eqe); 367 } 368 } 369 } 370 break; 371 372 case MLX4_EVENT_TYPE_CQ_ERROR: 373 mlx4_warn(dev, "CQ %s on CQN %06x\n", 374 eqe->event.cq_err.syndrome == 1 ? 375 "overrun" : "access violation", 376 be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff); 377 if (mlx4_is_master(dev)) { 378 ret = mlx4_get_slave_from_resource_id(dev, 379 RES_CQ, 380 be32_to_cpu(eqe->event.cq_err.cqn) 381 & 0xffffff, &slave); 382 if (ret && ret != -ENOENT) { 383 mlx4_dbg(dev, "CQ event %02x(%02x) on " 384 "EQ %d at index %u: could " 385 "not get slave id (%d)\n", 386 eqe->type, eqe->subtype, 387 eq->eqn, eq->cons_index, ret); 388 break; 389 } 390 391 if (!ret && slave != dev->caps.function) { 392 mlx4_slave_event(dev, slave, eqe); 393 break; 394 } 395 } 396 mlx4_cq_event(dev, 397 be32_to_cpu(eqe->event.cq_err.cqn) 398 & 0xffffff, 399 eqe->type); 400 break; 401 402 case MLX4_EVENT_TYPE_EQ_OVERFLOW: 403 mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn); 404 break; 405 406 case MLX4_EVENT_TYPE_COMM_CHANNEL: 407 if (!mlx4_is_master(dev)) { 408 mlx4_warn(dev, "Received comm channel event " 409 "for non master device\n"); 410 break; 411 } 412 memcpy(&priv->mfunc.master.comm_arm_bit_vector, 413 eqe->event.comm_channel_arm.bit_vec, 414 sizeof eqe->event.comm_channel_arm.bit_vec); 415 queue_work(priv->mfunc.master.comm_wq, 416 &priv->mfunc.master.comm_work); 417 break; 418 419 case MLX4_EVENT_TYPE_FLR_EVENT: 420 flr_slave = be32_to_cpu(eqe->event.flr_event.slave_id); 421 if (!mlx4_is_master(dev)) { 422 mlx4_warn(dev, "Non-master function received" 423 "FLR event\n"); 424 break; 425 } 426 427 mlx4_dbg(dev, "FLR event for slave: %d\n", flr_slave); 428 429 if (flr_slave > dev->num_slaves) { 430 mlx4_warn(dev, 431 "Got FLR for unknown function: %d\n", 432 flr_slave); 433 update_slave_state = 0; 434 } else 435 update_slave_state = 1; 436 437 spin_lock(&priv->mfunc.master.slave_state_lock); 438 if (update_slave_state) { 439 priv->mfunc.master.slave_state[flr_slave].active = false; 440 priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR; 441 priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1; 442 } 443 spin_unlock(&priv->mfunc.master.slave_state_lock); 444 queue_work(priv->mfunc.master.comm_wq, 445 &priv->mfunc.master.slave_flr_event_work); 446 break; 447 448 case MLX4_EVENT_TYPE_FATAL_WARNING: 449 if (eqe->subtype == MLX4_FATAL_WARNING_SUBTYPE_WARMING) { 450 if (mlx4_is_master(dev)) 451 for (i = 0; i < dev->num_slaves; i++) { 452 mlx4_dbg(dev, "%s: Sending " 453 "MLX4_FATAL_WARNING_SUBTYPE_WARMING" 454 " to slave: %d\n", __func__, i); 455 if (i == dev->caps.function) 456 continue; 457 mlx4_slave_event(dev, i, eqe); 458 } 459 mlx4_err(dev, "Temperature Threshold was reached! " 460 "Threshold: %d celsius degrees; " 461 "Current Temperature: %d\n", 462 be16_to_cpu(eqe->event.warming.warning_threshold), 463 be16_to_cpu(eqe->event.warming.current_temperature)); 464 } else 465 mlx4_warn(dev, "Unhandled event FATAL WARNING (%02x), " 466 "subtype %02x on EQ %d at index %u. owner=%x, " 467 "nent=0x%x, slave=%x, ownership=%s\n", 468 eqe->type, eqe->subtype, eq->eqn, 469 eq->cons_index, eqe->owner, eq->nent, 470 eqe->slave_id, 471 !!(eqe->owner & 0x80) ^ 472 !!(eq->cons_index & eq->nent) ? "HW" : "SW"); 473 474 break; 475 476 case MLX4_EVENT_TYPE_EEC_CATAS_ERROR: 477 case MLX4_EVENT_TYPE_ECC_DETECT: 478 default: 479 mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at " 480 "index %u. owner=%x, nent=0x%x, slave=%x, " 481 "ownership=%s\n", 482 eqe->type, eqe->subtype, eq->eqn, 483 eq->cons_index, eqe->owner, eq->nent, 484 eqe->slave_id, 485 !!(eqe->owner & 0x80) ^ 486 !!(eq->cons_index & eq->nent) ? "HW" : "SW"); 487 break; 488 }; 489 490 ++eq->cons_index; 491 eqes_found = 1; 492 ++set_ci; 493 494 /* 495 * The HCA will think the queue has overflowed if we 496 * don't tell it we've been processing events. We 497 * create our EQs with MLX4_NUM_SPARE_EQE extra 498 * entries, so we must update our consumer index at 499 * least that often. 500 */ 501 if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) { 502 eq_set_ci(eq, 0); 503 set_ci = 0; 504 } 505 } 506 507 eq_set_ci(eq, 1); 508 509 return eqes_found; 510 } 511 512 static irqreturn_t mlx4_interrupt(int irq, void *dev_ptr) 513 { 514 struct mlx4_dev *dev = dev_ptr; 515 struct mlx4_priv *priv = mlx4_priv(dev); 516 int work = 0; 517 int i; 518 519 writel(priv->eq_table.clr_mask, priv->eq_table.clr_int); 520 521 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) 522 work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]); 523 524 return IRQ_RETVAL(work); 525 } 526 527 static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr) 528 { 529 struct mlx4_eq *eq = eq_ptr; 530 struct mlx4_dev *dev = eq->dev; 531 532 mlx4_eq_int(dev, eq); 533 534 /* MSI-X vectors always belong to us */ 535 return IRQ_HANDLED; 536 } 537 538 int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave, 539 struct mlx4_vhcr *vhcr, 540 struct mlx4_cmd_mailbox *inbox, 541 struct mlx4_cmd_mailbox *outbox, 542 struct mlx4_cmd_info *cmd) 543 { 544 struct mlx4_priv *priv = mlx4_priv(dev); 545 struct mlx4_slave_event_eq_info *event_eq = 546 priv->mfunc.master.slave_state[slave].event_eq; 547 u32 in_modifier = vhcr->in_modifier; 548 u32 eqn = in_modifier & 0x1FF; 549 u64 in_param = vhcr->in_param; 550 int err = 0; 551 int i; 552 553 if (slave == dev->caps.function) 554 err = mlx4_cmd(dev, in_param, (in_modifier & 0x80000000) | eqn, 555 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, 556 MLX4_CMD_NATIVE); 557 if (!err) 558 for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) 559 if (in_param & (1LL << i)) 560 event_eq[i].eqn = in_modifier >> 31 ? -1 : eqn; 561 562 return err; 563 } 564 565 static int mlx4_MAP_EQ(struct mlx4_dev *dev, u64 event_mask, int unmap, 566 int eq_num) 567 { 568 return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num, 569 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B, 570 MLX4_CMD_WRAPPED); 571 } 572 573 static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 574 int eq_num) 575 { 576 return mlx4_cmd(dev, mailbox->dma, eq_num, 0, 577 MLX4_CMD_SW2HW_EQ, MLX4_CMD_TIME_CLASS_A, 578 MLX4_CMD_WRAPPED); 579 } 580 581 static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 582 int eq_num) 583 { 584 return mlx4_cmd_box(dev, 0, mailbox->dma, eq_num, 585 0, MLX4_CMD_HW2SW_EQ, MLX4_CMD_TIME_CLASS_A, 586 MLX4_CMD_WRAPPED); 587 } 588 589 static int mlx4_num_eq_uar(struct mlx4_dev *dev) 590 { 591 /* 592 * Each UAR holds 4 EQ doorbells. To figure out how many UARs 593 * we need to map, take the difference of highest index and 594 * the lowest index we'll use and add 1. 595 */ 596 return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs + 597 dev->caps.comp_pool)/4 - dev->caps.reserved_eqs/4 + 1; 598 } 599 600 static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq) 601 { 602 struct mlx4_priv *priv = mlx4_priv(dev); 603 int index; 604 605 index = eq->eqn / 4 - dev->caps.reserved_eqs / 4; 606 607 if (!priv->eq_table.uar_map[index]) { 608 priv->eq_table.uar_map[index] = 609 ioremap(pci_resource_start(dev->pdev, 2) + 610 ((eq->eqn / 4) << PAGE_SHIFT), 611 PAGE_SIZE); 612 if (!priv->eq_table.uar_map[index]) { 613 mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n", 614 eq->eqn); 615 return NULL; 616 } 617 } 618 619 return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4); 620 } 621 622 static int mlx4_create_eq(struct mlx4_dev *dev, int nent, 623 u8 intr, struct mlx4_eq *eq) 624 { 625 struct mlx4_priv *priv = mlx4_priv(dev); 626 struct mlx4_cmd_mailbox *mailbox; 627 struct mlx4_eq_context *eq_context; 628 int npages; 629 u64 *dma_list = NULL; 630 dma_addr_t t; 631 u64 mtt_addr; 632 int err = -ENOMEM; 633 int i; 634 635 eq->dev = dev; 636 eq->nent = roundup_pow_of_two(max(nent, 2)); 637 npages = PAGE_ALIGN(eq->nent * MLX4_EQ_ENTRY_SIZE) / PAGE_SIZE; 638 639 eq->page_list = kmalloc(npages * sizeof *eq->page_list, 640 GFP_KERNEL); 641 if (!eq->page_list) 642 goto err_out; 643 644 for (i = 0; i < npages; ++i) 645 eq->page_list[i].buf = NULL; 646 647 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); 648 if (!dma_list) 649 goto err_out_free; 650 651 mailbox = mlx4_alloc_cmd_mailbox(dev); 652 if (IS_ERR(mailbox)) 653 goto err_out_free; 654 eq_context = mailbox->buf; 655 656 for (i = 0; i < npages; ++i) { 657 eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, 658 PAGE_SIZE, &t, GFP_KERNEL); 659 if (!eq->page_list[i].buf) 660 goto err_out_free_pages; 661 662 dma_list[i] = t; 663 eq->page_list[i].map = t; 664 665 memset(eq->page_list[i].buf, 0, PAGE_SIZE); 666 } 667 668 eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap); 669 if (eq->eqn == -1) 670 goto err_out_free_pages; 671 672 eq->doorbell = mlx4_get_eq_uar(dev, eq); 673 if (!eq->doorbell) { 674 err = -ENOMEM; 675 goto err_out_free_eq; 676 } 677 678 err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt); 679 if (err) 680 goto err_out_free_eq; 681 682 err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list); 683 if (err) 684 goto err_out_free_mtt; 685 686 memset(eq_context, 0, sizeof *eq_context); 687 eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK | 688 MLX4_EQ_STATE_ARMED); 689 eq_context->log_eq_size = ilog2(eq->nent); 690 eq_context->intr = intr; 691 eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT; 692 693 mtt_addr = mlx4_mtt_addr(dev, &eq->mtt); 694 eq_context->mtt_base_addr_h = mtt_addr >> 32; 695 eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); 696 697 err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn); 698 if (err) { 699 mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err); 700 goto err_out_free_mtt; 701 } 702 703 kfree(dma_list); 704 mlx4_free_cmd_mailbox(dev, mailbox); 705 706 eq->cons_index = 0; 707 708 return err; 709 710 err_out_free_mtt: 711 mlx4_mtt_cleanup(dev, &eq->mtt); 712 713 err_out_free_eq: 714 mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn); 715 716 err_out_free_pages: 717 for (i = 0; i < npages; ++i) 718 if (eq->page_list[i].buf) 719 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 720 eq->page_list[i].buf, 721 eq->page_list[i].map); 722 723 mlx4_free_cmd_mailbox(dev, mailbox); 724 725 err_out_free: 726 kfree(eq->page_list); 727 kfree(dma_list); 728 729 err_out: 730 return err; 731 } 732 733 static void mlx4_free_eq(struct mlx4_dev *dev, 734 struct mlx4_eq *eq) 735 { 736 struct mlx4_priv *priv = mlx4_priv(dev); 737 struct mlx4_cmd_mailbox *mailbox; 738 int err; 739 int npages = PAGE_ALIGN(MLX4_EQ_ENTRY_SIZE * eq->nent) / PAGE_SIZE; 740 int i; 741 742 mailbox = mlx4_alloc_cmd_mailbox(dev); 743 if (IS_ERR(mailbox)) 744 return; 745 746 err = mlx4_HW2SW_EQ(dev, mailbox, eq->eqn); 747 if (err) 748 mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err); 749 750 if (0) { 751 mlx4_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn); 752 for (i = 0; i < sizeof (struct mlx4_eq_context) / 4; ++i) { 753 if (i % 4 == 0) 754 pr_cont("[%02x] ", i * 4); 755 pr_cont(" %08x", be32_to_cpup(mailbox->buf + i * 4)); 756 if ((i + 1) % 4 == 0) 757 pr_cont("\n"); 758 } 759 } 760 761 mlx4_mtt_cleanup(dev, &eq->mtt); 762 for (i = 0; i < npages; ++i) 763 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 764 eq->page_list[i].buf, 765 eq->page_list[i].map); 766 767 kfree(eq->page_list); 768 mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn); 769 mlx4_free_cmd_mailbox(dev, mailbox); 770 } 771 772 static void mlx4_free_irqs(struct mlx4_dev *dev) 773 { 774 struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table; 775 struct mlx4_priv *priv = mlx4_priv(dev); 776 int i, vec; 777 778 if (eq_table->have_irq) 779 free_irq(dev->pdev->irq, dev); 780 781 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) 782 if (eq_table->eq[i].have_irq) { 783 free_irq(eq_table->eq[i].irq, eq_table->eq + i); 784 eq_table->eq[i].have_irq = 0; 785 } 786 787 for (i = 0; i < dev->caps.comp_pool; i++) { 788 /* 789 * Freeing the assigned irq's 790 * all bits should be 0, but we need to validate 791 */ 792 if (priv->msix_ctl.pool_bm & 1ULL << i) { 793 /* NO need protecting*/ 794 vec = dev->caps.num_comp_vectors + 1 + i; 795 free_irq(priv->eq_table.eq[vec].irq, 796 &priv->eq_table.eq[vec]); 797 } 798 } 799 800 801 kfree(eq_table->irq_names); 802 } 803 804 static int mlx4_map_clr_int(struct mlx4_dev *dev) 805 { 806 struct mlx4_priv *priv = mlx4_priv(dev); 807 808 priv->clr_base = ioremap(pci_resource_start(dev->pdev, priv->fw.clr_int_bar) + 809 priv->fw.clr_int_base, MLX4_CLR_INT_SIZE); 810 if (!priv->clr_base) { 811 mlx4_err(dev, "Couldn't map interrupt clear register, aborting.\n"); 812 return -ENOMEM; 813 } 814 815 return 0; 816 } 817 818 static void mlx4_unmap_clr_int(struct mlx4_dev *dev) 819 { 820 struct mlx4_priv *priv = mlx4_priv(dev); 821 822 iounmap(priv->clr_base); 823 } 824 825 int mlx4_alloc_eq_table(struct mlx4_dev *dev) 826 { 827 struct mlx4_priv *priv = mlx4_priv(dev); 828 829 priv->eq_table.eq = kcalloc(dev->caps.num_eqs - dev->caps.reserved_eqs, 830 sizeof *priv->eq_table.eq, GFP_KERNEL); 831 if (!priv->eq_table.eq) 832 return -ENOMEM; 833 834 return 0; 835 } 836 837 void mlx4_free_eq_table(struct mlx4_dev *dev) 838 { 839 kfree(mlx4_priv(dev)->eq_table.eq); 840 } 841 842 int mlx4_init_eq_table(struct mlx4_dev *dev) 843 { 844 struct mlx4_priv *priv = mlx4_priv(dev); 845 int err; 846 int i; 847 848 priv->eq_table.uar_map = kcalloc(mlx4_num_eq_uar(dev), 849 sizeof *priv->eq_table.uar_map, 850 GFP_KERNEL); 851 if (!priv->eq_table.uar_map) { 852 err = -ENOMEM; 853 goto err_out_free; 854 } 855 856 err = mlx4_bitmap_init(&priv->eq_table.bitmap, dev->caps.num_eqs, 857 dev->caps.num_eqs - 1, dev->caps.reserved_eqs, 0); 858 if (err) 859 goto err_out_free; 860 861 for (i = 0; i < mlx4_num_eq_uar(dev); ++i) 862 priv->eq_table.uar_map[i] = NULL; 863 864 if (!mlx4_is_slave(dev)) { 865 err = mlx4_map_clr_int(dev); 866 if (err) 867 goto err_out_bitmap; 868 869 priv->eq_table.clr_mask = 870 swab32(1 << (priv->eq_table.inta_pin & 31)); 871 priv->eq_table.clr_int = priv->clr_base + 872 (priv->eq_table.inta_pin < 32 ? 4 : 0); 873 } 874 875 priv->eq_table.irq_names = 876 kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1 + 877 dev->caps.comp_pool), 878 GFP_KERNEL); 879 if (!priv->eq_table.irq_names) { 880 err = -ENOMEM; 881 goto err_out_bitmap; 882 } 883 884 for (i = 0; i < dev->caps.num_comp_vectors; ++i) { 885 err = mlx4_create_eq(dev, dev->caps.num_cqs - 886 dev->caps.reserved_cqs + 887 MLX4_NUM_SPARE_EQE, 888 (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, 889 &priv->eq_table.eq[i]); 890 if (err) { 891 --i; 892 goto err_out_unmap; 893 } 894 } 895 896 err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE, 897 (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0, 898 &priv->eq_table.eq[dev->caps.num_comp_vectors]); 899 if (err) 900 goto err_out_comp; 901 902 /*if additional completion vectors poolsize is 0 this loop will not run*/ 903 for (i = dev->caps.num_comp_vectors + 1; 904 i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) { 905 906 err = mlx4_create_eq(dev, dev->caps.num_cqs - 907 dev->caps.reserved_cqs + 908 MLX4_NUM_SPARE_EQE, 909 (dev->flags & MLX4_FLAG_MSI_X) ? i : 0, 910 &priv->eq_table.eq[i]); 911 if (err) { 912 --i; 913 goto err_out_unmap; 914 } 915 } 916 917 918 if (dev->flags & MLX4_FLAG_MSI_X) { 919 const char *eq_name; 920 921 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) { 922 if (i < dev->caps.num_comp_vectors) { 923 snprintf(priv->eq_table.irq_names + 924 i * MLX4_IRQNAME_SIZE, 925 MLX4_IRQNAME_SIZE, 926 "mlx4-comp-%d@pci:%s", i, 927 pci_name(dev->pdev)); 928 } else { 929 snprintf(priv->eq_table.irq_names + 930 i * MLX4_IRQNAME_SIZE, 931 MLX4_IRQNAME_SIZE, 932 "mlx4-async@pci:%s", 933 pci_name(dev->pdev)); 934 } 935 936 eq_name = priv->eq_table.irq_names + 937 i * MLX4_IRQNAME_SIZE; 938 err = request_irq(priv->eq_table.eq[i].irq, 939 mlx4_msi_x_interrupt, 0, eq_name, 940 priv->eq_table.eq + i); 941 if (err) 942 goto err_out_async; 943 944 priv->eq_table.eq[i].have_irq = 1; 945 } 946 } else { 947 snprintf(priv->eq_table.irq_names, 948 MLX4_IRQNAME_SIZE, 949 DRV_NAME "@pci:%s", 950 pci_name(dev->pdev)); 951 err = request_irq(dev->pdev->irq, mlx4_interrupt, 952 IRQF_SHARED, priv->eq_table.irq_names, dev); 953 if (err) 954 goto err_out_async; 955 956 priv->eq_table.have_irq = 1; 957 } 958 959 err = mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 0, 960 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 961 if (err) 962 mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n", 963 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn, err); 964 965 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) 966 eq_set_ci(&priv->eq_table.eq[i], 1); 967 968 return 0; 969 970 err_out_async: 971 mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]); 972 973 err_out_comp: 974 i = dev->caps.num_comp_vectors - 1; 975 976 err_out_unmap: 977 while (i >= 0) { 978 mlx4_free_eq(dev, &priv->eq_table.eq[i]); 979 --i; 980 } 981 if (!mlx4_is_slave(dev)) 982 mlx4_unmap_clr_int(dev); 983 mlx4_free_irqs(dev); 984 985 err_out_bitmap: 986 mlx4_bitmap_cleanup(&priv->eq_table.bitmap); 987 988 err_out_free: 989 kfree(priv->eq_table.uar_map); 990 991 return err; 992 } 993 994 void mlx4_cleanup_eq_table(struct mlx4_dev *dev) 995 { 996 struct mlx4_priv *priv = mlx4_priv(dev); 997 int i; 998 999 mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 1, 1000 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 1001 1002 mlx4_free_irqs(dev); 1003 1004 for (i = 0; i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) 1005 mlx4_free_eq(dev, &priv->eq_table.eq[i]); 1006 1007 if (!mlx4_is_slave(dev)) 1008 mlx4_unmap_clr_int(dev); 1009 1010 for (i = 0; i < mlx4_num_eq_uar(dev); ++i) 1011 if (priv->eq_table.uar_map[i]) 1012 iounmap(priv->eq_table.uar_map[i]); 1013 1014 mlx4_bitmap_cleanup(&priv->eq_table.bitmap); 1015 1016 kfree(priv->eq_table.uar_map); 1017 } 1018 1019 /* A test that verifies that we can accept interrupts on all 1020 * the irq vectors of the device. 1021 * Interrupts are checked using the NOP command. 1022 */ 1023 int mlx4_test_interrupts(struct mlx4_dev *dev) 1024 { 1025 struct mlx4_priv *priv = mlx4_priv(dev); 1026 int i; 1027 int err; 1028 1029 err = mlx4_NOP(dev); 1030 /* When not in MSI_X, there is only one irq to check */ 1031 if (!(dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(dev)) 1032 return err; 1033 1034 /* A loop over all completion vectors, for each vector we will check 1035 * whether it works by mapping command completions to that vector 1036 * and performing a NOP command 1037 */ 1038 for(i = 0; !err && (i < dev->caps.num_comp_vectors); ++i) { 1039 /* Temporary use polling for command completions */ 1040 mlx4_cmd_use_polling(dev); 1041 1042 /* Map the new eq to handle all asyncronous events */ 1043 err = mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 0, 1044 priv->eq_table.eq[i].eqn); 1045 if (err) { 1046 mlx4_warn(dev, "Failed mapping eq for interrupt test\n"); 1047 mlx4_cmd_use_events(dev); 1048 break; 1049 } 1050 1051 /* Go back to using events */ 1052 mlx4_cmd_use_events(dev); 1053 err = mlx4_NOP(dev); 1054 } 1055 1056 /* Return to default */ 1057 mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 0, 1058 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn); 1059 return err; 1060 } 1061 EXPORT_SYMBOL(mlx4_test_interrupts); 1062 1063 int mlx4_assign_eq(struct mlx4_dev *dev, char* name, int * vector) 1064 { 1065 1066 struct mlx4_priv *priv = mlx4_priv(dev); 1067 int vec = 0, err = 0, i; 1068 1069 mutex_lock(&priv->msix_ctl.pool_lock); 1070 for (i = 0; !vec && i < dev->caps.comp_pool; i++) { 1071 if (~priv->msix_ctl.pool_bm & 1ULL << i) { 1072 priv->msix_ctl.pool_bm |= 1ULL << i; 1073 vec = dev->caps.num_comp_vectors + 1 + i; 1074 snprintf(priv->eq_table.irq_names + 1075 vec * MLX4_IRQNAME_SIZE, 1076 MLX4_IRQNAME_SIZE, "%s", name); 1077 err = request_irq(priv->eq_table.eq[vec].irq, 1078 mlx4_msi_x_interrupt, 0, 1079 &priv->eq_table.irq_names[vec<<5], 1080 priv->eq_table.eq + vec); 1081 if (err) { 1082 /*zero out bit by fliping it*/ 1083 priv->msix_ctl.pool_bm ^= 1 << i; 1084 vec = 0; 1085 continue; 1086 /*we dont want to break here*/ 1087 } 1088 eq_set_ci(&priv->eq_table.eq[vec], 1); 1089 } 1090 } 1091 mutex_unlock(&priv->msix_ctl.pool_lock); 1092 1093 if (vec) { 1094 *vector = vec; 1095 } else { 1096 *vector = 0; 1097 err = (i == dev->caps.comp_pool) ? -ENOSPC : err; 1098 } 1099 return err; 1100 } 1101 EXPORT_SYMBOL(mlx4_assign_eq); 1102 1103 void mlx4_release_eq(struct mlx4_dev *dev, int vec) 1104 { 1105 struct mlx4_priv *priv = mlx4_priv(dev); 1106 /*bm index*/ 1107 int i = vec - dev->caps.num_comp_vectors - 1; 1108 1109 if (likely(i >= 0)) { 1110 /*sanity check , making sure were not trying to free irq's 1111 Belonging to a legacy EQ*/ 1112 mutex_lock(&priv->msix_ctl.pool_lock); 1113 if (priv->msix_ctl.pool_bm & 1ULL << i) { 1114 free_irq(priv->eq_table.eq[vec].irq, 1115 &priv->eq_table.eq[vec]); 1116 priv->msix_ctl.pool_bm &= ~(1ULL << i); 1117 } 1118 mutex_unlock(&priv->msix_ctl.pool_lock); 1119 } 1120 1121 } 1122 EXPORT_SYMBOL(mlx4_release_eq); 1123 1124