1 /* 2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/highmem.h> 34 #include <linux/errno.h> 35 #include <linux/pci.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/slab.h> 38 #include <linux/delay.h> 39 #include <linux/random.h> 40 #include <linux/mlx5/driver.h> 41 #include <linux/mlx5/eq.h> 42 #include <linux/debugfs.h> 43 44 #include "mlx5_core.h" 45 #include "lib/eq.h" 46 #include "lib/tout.h" 47 #define CREATE_TRACE_POINTS 48 #include "diag/cmd_tracepoint.h" 49 50 struct mlx5_ifc_mbox_out_bits { 51 u8 status[0x8]; 52 u8 reserved_at_8[0x18]; 53 54 u8 syndrome[0x20]; 55 56 u8 reserved_at_40[0x40]; 57 }; 58 59 struct mlx5_ifc_mbox_in_bits { 60 u8 opcode[0x10]; 61 u8 uid[0x10]; 62 63 u8 reserved_at_20[0x10]; 64 u8 op_mod[0x10]; 65 66 u8 reserved_at_40[0x40]; 67 }; 68 69 enum { 70 CMD_IF_REV = 5, 71 }; 72 73 enum { 74 CMD_MODE_POLLING, 75 CMD_MODE_EVENTS 76 }; 77 78 enum { 79 MLX5_CMD_DELIVERY_STAT_OK = 0x0, 80 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1, 81 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2, 82 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3, 83 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4, 84 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5, 85 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6, 86 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7, 87 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8, 88 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9, 89 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10, 90 }; 91 92 static u16 in_to_opcode(void *in) 93 { 94 return MLX5_GET(mbox_in, in, opcode); 95 } 96 97 /* Returns true for opcodes that might be triggered very frequently and throttle 98 * the command interface. Limit their command slots usage. 99 */ 100 static bool mlx5_cmd_is_throttle_opcode(u16 op) 101 { 102 switch (op) { 103 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 104 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: 105 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 106 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 107 case MLX5_CMD_OP_SYNC_CRYPTO: 108 return true; 109 } 110 return false; 111 } 112 113 static struct mlx5_cmd_work_ent * 114 cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in, 115 struct mlx5_cmd_msg *out, void *uout, int uout_size, 116 mlx5_cmd_cbk_t cbk, void *context, int page_queue) 117 { 118 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL; 119 struct mlx5_cmd_work_ent *ent; 120 121 ent = kzalloc(sizeof(*ent), alloc_flags); 122 if (!ent) 123 return ERR_PTR(-ENOMEM); 124 125 ent->idx = -EINVAL; 126 ent->in = in; 127 ent->out = out; 128 ent->uout = uout; 129 ent->uout_size = uout_size; 130 ent->callback = cbk; 131 ent->context = context; 132 ent->cmd = cmd; 133 ent->page_queue = page_queue; 134 ent->op = in_to_opcode(in->first.data); 135 refcount_set(&ent->refcnt, 1); 136 137 return ent; 138 } 139 140 static void cmd_free_ent(struct mlx5_cmd_work_ent *ent) 141 { 142 kfree(ent); 143 } 144 145 static u8 alloc_token(struct mlx5_cmd *cmd) 146 { 147 u8 token; 148 149 spin_lock(&cmd->token_lock); 150 cmd->token++; 151 if (cmd->token == 0) 152 cmd->token++; 153 token = cmd->token; 154 spin_unlock(&cmd->token_lock); 155 156 return token; 157 } 158 159 static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent) 160 { 161 unsigned long flags; 162 int ret; 163 164 spin_lock_irqsave(&cmd->alloc_lock, flags); 165 ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds); 166 if (ret < cmd->vars.max_reg_cmds) { 167 clear_bit(ret, &cmd->vars.bitmask); 168 ent->idx = ret; 169 cmd->ent_arr[ent->idx] = ent; 170 } 171 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 172 173 return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM; 174 } 175 176 static void cmd_free_index(struct mlx5_cmd *cmd, int idx) 177 { 178 lockdep_assert_held(&cmd->alloc_lock); 179 set_bit(idx, &cmd->vars.bitmask); 180 } 181 182 static void cmd_ent_get(struct mlx5_cmd_work_ent *ent) 183 { 184 refcount_inc(&ent->refcnt); 185 } 186 187 static void cmd_ent_put(struct mlx5_cmd_work_ent *ent) 188 { 189 struct mlx5_cmd *cmd = ent->cmd; 190 unsigned long flags; 191 192 spin_lock_irqsave(&cmd->alloc_lock, flags); 193 if (!refcount_dec_and_test(&ent->refcnt)) 194 goto out; 195 196 if (ent->idx >= 0) { 197 cmd_free_index(cmd, ent->idx); 198 up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem); 199 } 200 201 cmd_free_ent(ent); 202 out: 203 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 204 } 205 206 static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx) 207 { 208 return cmd->cmd_buf + (idx << cmd->vars.log_stride); 209 } 210 211 static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg) 212 { 213 int size = msg->len; 214 int blen = size - min_t(int, sizeof(msg->first.data), size); 215 216 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE); 217 } 218 219 static u8 xor8_buf(void *buf, size_t offset, int len) 220 { 221 u8 *ptr = buf; 222 u8 sum = 0; 223 int i; 224 int end = len + offset; 225 226 for (i = offset; i < end; i++) 227 sum ^= ptr[i]; 228 229 return sum; 230 } 231 232 static int verify_block_sig(struct mlx5_cmd_prot_block *block) 233 { 234 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0); 235 int xor_len = sizeof(*block) - sizeof(block->data) - 1; 236 237 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff) 238 return -EHWPOISON; 239 240 if (xor8_buf(block, 0, sizeof(*block)) != 0xff) 241 return -EHWPOISON; 242 243 return 0; 244 } 245 246 static void calc_block_sig(struct mlx5_cmd_prot_block *block) 247 { 248 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2; 249 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0); 250 251 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len); 252 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1); 253 } 254 255 static void calc_chain_sig(struct mlx5_cmd_msg *msg) 256 { 257 struct mlx5_cmd_mailbox *next = msg->next; 258 int n = mlx5_calc_cmd_blocks(msg); 259 int i = 0; 260 261 for (i = 0; i < n && next; i++) { 262 calc_block_sig(next->buf); 263 next = next->next; 264 } 265 } 266 267 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum) 268 { 269 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay)); 270 if (csum) { 271 calc_chain_sig(ent->in); 272 calc_chain_sig(ent->out); 273 } 274 } 275 276 static void poll_timeout(struct mlx5_cmd_work_ent *ent) 277 { 278 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd); 279 u64 cmd_to_ms = mlx5_tout_ms(dev, CMD); 280 unsigned long poll_end; 281 u8 own; 282 283 poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000); 284 285 do { 286 own = READ_ONCE(ent->lay->status_own); 287 if (!(own & CMD_OWNER_HW)) { 288 ent->ret = 0; 289 return; 290 } 291 cond_resched(); 292 } while (time_before(jiffies, poll_end)); 293 294 ent->ret = -ETIMEDOUT; 295 } 296 297 static int verify_signature(struct mlx5_cmd_work_ent *ent) 298 { 299 struct mlx5_cmd_mailbox *next = ent->out->next; 300 int n = mlx5_calc_cmd_blocks(ent->out); 301 int err; 302 u8 sig; 303 int i = 0; 304 305 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay)); 306 if (sig != 0xff) 307 return -EHWPOISON; 308 309 for (i = 0; i < n && next; i++) { 310 err = verify_block_sig(next->buf); 311 if (err) 312 return -EHWPOISON; 313 314 next = next->next; 315 } 316 317 return 0; 318 } 319 320 static void dump_buf(void *buf, int size, int data_only, int offset, int idx) 321 { 322 __be32 *p = buf; 323 int i; 324 325 for (i = 0; i < size; i += 16) { 326 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset, 327 be32_to_cpu(p[0]), be32_to_cpu(p[1]), 328 be32_to_cpu(p[2]), be32_to_cpu(p[3])); 329 p += 4; 330 offset += 16; 331 } 332 if (!data_only) 333 pr_debug("\n"); 334 } 335 336 static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, 337 u32 *synd, u8 *status) 338 { 339 *synd = 0; 340 *status = 0; 341 342 switch (op) { 343 case MLX5_CMD_OP_TEARDOWN_HCA: 344 case MLX5_CMD_OP_DISABLE_HCA: 345 case MLX5_CMD_OP_MANAGE_PAGES: 346 case MLX5_CMD_OP_DESTROY_MKEY: 347 case MLX5_CMD_OP_DESTROY_EQ: 348 case MLX5_CMD_OP_DESTROY_CQ: 349 case MLX5_CMD_OP_DESTROY_QP: 350 case MLX5_CMD_OP_DESTROY_PSV: 351 case MLX5_CMD_OP_DESTROY_SRQ: 352 case MLX5_CMD_OP_DESTROY_XRC_SRQ: 353 case MLX5_CMD_OP_DESTROY_XRQ: 354 case MLX5_CMD_OP_DESTROY_DCT: 355 case MLX5_CMD_OP_DEALLOC_Q_COUNTER: 356 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT: 357 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT: 358 case MLX5_CMD_OP_DEALLOC_PD: 359 case MLX5_CMD_OP_DEALLOC_UAR: 360 case MLX5_CMD_OP_DETACH_FROM_MCG: 361 case MLX5_CMD_OP_DEALLOC_XRCD: 362 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN: 363 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT: 364 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY: 365 case MLX5_CMD_OP_DESTROY_LAG: 366 case MLX5_CMD_OP_DESTROY_VPORT_LAG: 367 case MLX5_CMD_OP_DESTROY_TIR: 368 case MLX5_CMD_OP_DESTROY_SQ: 369 case MLX5_CMD_OP_DESTROY_RQ: 370 case MLX5_CMD_OP_DESTROY_RMP: 371 case MLX5_CMD_OP_DESTROY_TIS: 372 case MLX5_CMD_OP_DESTROY_RQT: 373 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 374 case MLX5_CMD_OP_DESTROY_FLOW_GROUP: 375 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY: 376 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER: 377 case MLX5_CMD_OP_2ERR_QP: 378 case MLX5_CMD_OP_2RST_QP: 379 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT: 380 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 381 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 382 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT: 383 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT: 384 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT: 385 case MLX5_CMD_OP_FPGA_DESTROY_QP: 386 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: 387 case MLX5_CMD_OP_DEALLOC_MEMIC: 388 case MLX5_CMD_OP_PAGE_FAULT_RESUME: 389 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS: 390 case MLX5_CMD_OP_DEALLOC_SF: 391 case MLX5_CMD_OP_DESTROY_UCTX: 392 case MLX5_CMD_OP_DESTROY_UMEM: 393 case MLX5_CMD_OP_MODIFY_RQT: 394 return MLX5_CMD_STAT_OK; 395 396 case MLX5_CMD_OP_QUERY_HCA_CAP: 397 case MLX5_CMD_OP_QUERY_ADAPTER: 398 case MLX5_CMD_OP_INIT_HCA: 399 case MLX5_CMD_OP_ENABLE_HCA: 400 case MLX5_CMD_OP_QUERY_PAGES: 401 case MLX5_CMD_OP_SET_HCA_CAP: 402 case MLX5_CMD_OP_QUERY_ISSI: 403 case MLX5_CMD_OP_SET_ISSI: 404 case MLX5_CMD_OP_CREATE_MKEY: 405 case MLX5_CMD_OP_QUERY_MKEY: 406 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS: 407 case MLX5_CMD_OP_CREATE_EQ: 408 case MLX5_CMD_OP_QUERY_EQ: 409 case MLX5_CMD_OP_GEN_EQE: 410 case MLX5_CMD_OP_CREATE_CQ: 411 case MLX5_CMD_OP_QUERY_CQ: 412 case MLX5_CMD_OP_MODIFY_CQ: 413 case MLX5_CMD_OP_CREATE_QP: 414 case MLX5_CMD_OP_RST2INIT_QP: 415 case MLX5_CMD_OP_INIT2RTR_QP: 416 case MLX5_CMD_OP_RTR2RTS_QP: 417 case MLX5_CMD_OP_RTS2RTS_QP: 418 case MLX5_CMD_OP_SQERR2RTS_QP: 419 case MLX5_CMD_OP_QUERY_QP: 420 case MLX5_CMD_OP_SQD_RTS_QP: 421 case MLX5_CMD_OP_INIT2INIT_QP: 422 case MLX5_CMD_OP_CREATE_PSV: 423 case MLX5_CMD_OP_CREATE_SRQ: 424 case MLX5_CMD_OP_QUERY_SRQ: 425 case MLX5_CMD_OP_ARM_RQ: 426 case MLX5_CMD_OP_CREATE_XRC_SRQ: 427 case MLX5_CMD_OP_QUERY_XRC_SRQ: 428 case MLX5_CMD_OP_ARM_XRC_SRQ: 429 case MLX5_CMD_OP_CREATE_XRQ: 430 case MLX5_CMD_OP_QUERY_XRQ: 431 case MLX5_CMD_OP_ARM_XRQ: 432 case MLX5_CMD_OP_CREATE_DCT: 433 case MLX5_CMD_OP_DRAIN_DCT: 434 case MLX5_CMD_OP_QUERY_DCT: 435 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 436 case MLX5_CMD_OP_QUERY_VPORT_STATE: 437 case MLX5_CMD_OP_MODIFY_VPORT_STATE: 438 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: 439 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT: 440 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT: 441 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS: 442 case MLX5_CMD_OP_SET_ROCE_ADDRESS: 443 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 444 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT: 445 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID: 446 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY: 447 case MLX5_CMD_OP_QUERY_VNIC_ENV: 448 case MLX5_CMD_OP_QUERY_VPORT_COUNTER: 449 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 450 case MLX5_CMD_OP_QUERY_Q_COUNTER: 451 case MLX5_CMD_OP_SET_MONITOR_COUNTER: 452 case MLX5_CMD_OP_ARM_MONITOR_COUNTER: 453 case MLX5_CMD_OP_SET_PP_RATE_LIMIT: 454 case MLX5_CMD_OP_QUERY_RATE_LIMIT: 455 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 456 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 457 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 458 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT: 459 case MLX5_CMD_OP_ALLOC_PD: 460 case MLX5_CMD_OP_ALLOC_UAR: 461 case MLX5_CMD_OP_CONFIG_INT_MODERATION: 462 case MLX5_CMD_OP_ACCESS_REG: 463 case MLX5_CMD_OP_ATTACH_TO_MCG: 464 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG: 465 case MLX5_CMD_OP_MAD_IFC: 466 case MLX5_CMD_OP_QUERY_MAD_DEMUX: 467 case MLX5_CMD_OP_SET_MAD_DEMUX: 468 case MLX5_CMD_OP_NOP: 469 case MLX5_CMD_OP_ALLOC_XRCD: 470 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 471 case MLX5_CMD_OP_QUERY_CONG_STATUS: 472 case MLX5_CMD_OP_MODIFY_CONG_STATUS: 473 case MLX5_CMD_OP_QUERY_CONG_PARAMS: 474 case MLX5_CMD_OP_MODIFY_CONG_PARAMS: 475 case MLX5_CMD_OP_QUERY_CONG_STATISTICS: 476 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 477 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 478 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 479 case MLX5_CMD_OP_CREATE_LAG: 480 case MLX5_CMD_OP_MODIFY_LAG: 481 case MLX5_CMD_OP_QUERY_LAG: 482 case MLX5_CMD_OP_CREATE_VPORT_LAG: 483 case MLX5_CMD_OP_CREATE_TIR: 484 case MLX5_CMD_OP_MODIFY_TIR: 485 case MLX5_CMD_OP_QUERY_TIR: 486 case MLX5_CMD_OP_CREATE_SQ: 487 case MLX5_CMD_OP_MODIFY_SQ: 488 case MLX5_CMD_OP_QUERY_SQ: 489 case MLX5_CMD_OP_CREATE_RQ: 490 case MLX5_CMD_OP_MODIFY_RQ: 491 case MLX5_CMD_OP_QUERY_RQ: 492 case MLX5_CMD_OP_CREATE_RMP: 493 case MLX5_CMD_OP_MODIFY_RMP: 494 case MLX5_CMD_OP_QUERY_RMP: 495 case MLX5_CMD_OP_CREATE_TIS: 496 case MLX5_CMD_OP_MODIFY_TIS: 497 case MLX5_CMD_OP_QUERY_TIS: 498 case MLX5_CMD_OP_CREATE_RQT: 499 case MLX5_CMD_OP_QUERY_RQT: 500 501 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 502 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 503 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 504 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 505 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 506 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 507 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 508 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 509 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 510 case MLX5_CMD_OP_FPGA_CREATE_QP: 511 case MLX5_CMD_OP_FPGA_MODIFY_QP: 512 case MLX5_CMD_OP_FPGA_QUERY_QP: 513 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS: 514 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 515 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 516 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 517 case MLX5_CMD_OP_CREATE_UCTX: 518 case MLX5_CMD_OP_CREATE_UMEM: 519 case MLX5_CMD_OP_ALLOC_MEMIC: 520 case MLX5_CMD_OP_MODIFY_XRQ: 521 case MLX5_CMD_OP_RELEASE_XRQ_ERROR: 522 case MLX5_CMD_OP_QUERY_VHCA_STATE: 523 case MLX5_CMD_OP_MODIFY_VHCA_STATE: 524 case MLX5_CMD_OP_ALLOC_SF: 525 case MLX5_CMD_OP_SUSPEND_VHCA: 526 case MLX5_CMD_OP_RESUME_VHCA: 527 case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE: 528 case MLX5_CMD_OP_SAVE_VHCA_STATE: 529 case MLX5_CMD_OP_LOAD_VHCA_STATE: 530 case MLX5_CMD_OP_SYNC_CRYPTO: 531 *status = MLX5_DRIVER_STATUS_ABORTED; 532 *synd = MLX5_DRIVER_SYND; 533 return -ENOLINK; 534 default: 535 mlx5_core_err(dev, "Unknown FW command (%d)\n", op); 536 return -EINVAL; 537 } 538 } 539 540 const char *mlx5_command_str(int command) 541 { 542 #define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd 543 544 switch (command) { 545 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP); 546 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER); 547 MLX5_COMMAND_STR_CASE(INIT_HCA); 548 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA); 549 MLX5_COMMAND_STR_CASE(ENABLE_HCA); 550 MLX5_COMMAND_STR_CASE(DISABLE_HCA); 551 MLX5_COMMAND_STR_CASE(QUERY_PAGES); 552 MLX5_COMMAND_STR_CASE(MANAGE_PAGES); 553 MLX5_COMMAND_STR_CASE(SET_HCA_CAP); 554 MLX5_COMMAND_STR_CASE(QUERY_ISSI); 555 MLX5_COMMAND_STR_CASE(SET_ISSI); 556 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION); 557 MLX5_COMMAND_STR_CASE(CREATE_MKEY); 558 MLX5_COMMAND_STR_CASE(QUERY_MKEY); 559 MLX5_COMMAND_STR_CASE(DESTROY_MKEY); 560 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS); 561 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME); 562 MLX5_COMMAND_STR_CASE(CREATE_EQ); 563 MLX5_COMMAND_STR_CASE(DESTROY_EQ); 564 MLX5_COMMAND_STR_CASE(QUERY_EQ); 565 MLX5_COMMAND_STR_CASE(GEN_EQE); 566 MLX5_COMMAND_STR_CASE(CREATE_CQ); 567 MLX5_COMMAND_STR_CASE(DESTROY_CQ); 568 MLX5_COMMAND_STR_CASE(QUERY_CQ); 569 MLX5_COMMAND_STR_CASE(MODIFY_CQ); 570 MLX5_COMMAND_STR_CASE(CREATE_QP); 571 MLX5_COMMAND_STR_CASE(DESTROY_QP); 572 MLX5_COMMAND_STR_CASE(RST2INIT_QP); 573 MLX5_COMMAND_STR_CASE(INIT2RTR_QP); 574 MLX5_COMMAND_STR_CASE(RTR2RTS_QP); 575 MLX5_COMMAND_STR_CASE(RTS2RTS_QP); 576 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP); 577 MLX5_COMMAND_STR_CASE(2ERR_QP); 578 MLX5_COMMAND_STR_CASE(2RST_QP); 579 MLX5_COMMAND_STR_CASE(QUERY_QP); 580 MLX5_COMMAND_STR_CASE(SQD_RTS_QP); 581 MLX5_COMMAND_STR_CASE(INIT2INIT_QP); 582 MLX5_COMMAND_STR_CASE(CREATE_PSV); 583 MLX5_COMMAND_STR_CASE(DESTROY_PSV); 584 MLX5_COMMAND_STR_CASE(CREATE_SRQ); 585 MLX5_COMMAND_STR_CASE(DESTROY_SRQ); 586 MLX5_COMMAND_STR_CASE(QUERY_SRQ); 587 MLX5_COMMAND_STR_CASE(ARM_RQ); 588 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ); 589 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ); 590 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ); 591 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ); 592 MLX5_COMMAND_STR_CASE(CREATE_DCT); 593 MLX5_COMMAND_STR_CASE(DESTROY_DCT); 594 MLX5_COMMAND_STR_CASE(DRAIN_DCT); 595 MLX5_COMMAND_STR_CASE(QUERY_DCT); 596 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION); 597 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE); 598 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE); 599 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT); 600 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT); 601 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT); 602 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT); 603 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS); 604 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS); 605 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT); 606 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT); 607 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID); 608 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY); 609 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV); 610 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER); 611 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER); 612 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER); 613 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER); 614 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER); 615 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER); 616 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT); 617 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT); 618 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT); 619 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT); 620 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT); 621 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT); 622 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT); 623 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT); 624 MLX5_COMMAND_STR_CASE(ALLOC_PD); 625 MLX5_COMMAND_STR_CASE(DEALLOC_PD); 626 MLX5_COMMAND_STR_CASE(ALLOC_UAR); 627 MLX5_COMMAND_STR_CASE(DEALLOC_UAR); 628 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION); 629 MLX5_COMMAND_STR_CASE(ACCESS_REG); 630 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG); 631 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG); 632 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG); 633 MLX5_COMMAND_STR_CASE(MAD_IFC); 634 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX); 635 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX); 636 MLX5_COMMAND_STR_CASE(NOP); 637 MLX5_COMMAND_STR_CASE(ALLOC_XRCD); 638 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD); 639 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN); 640 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN); 641 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS); 642 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS); 643 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS); 644 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS); 645 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS); 646 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT); 647 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT); 648 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY); 649 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY); 650 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY); 651 MLX5_COMMAND_STR_CASE(SET_WOL_ROL); 652 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL); 653 MLX5_COMMAND_STR_CASE(CREATE_LAG); 654 MLX5_COMMAND_STR_CASE(MODIFY_LAG); 655 MLX5_COMMAND_STR_CASE(QUERY_LAG); 656 MLX5_COMMAND_STR_CASE(DESTROY_LAG); 657 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG); 658 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG); 659 MLX5_COMMAND_STR_CASE(CREATE_TIR); 660 MLX5_COMMAND_STR_CASE(MODIFY_TIR); 661 MLX5_COMMAND_STR_CASE(DESTROY_TIR); 662 MLX5_COMMAND_STR_CASE(QUERY_TIR); 663 MLX5_COMMAND_STR_CASE(CREATE_SQ); 664 MLX5_COMMAND_STR_CASE(MODIFY_SQ); 665 MLX5_COMMAND_STR_CASE(DESTROY_SQ); 666 MLX5_COMMAND_STR_CASE(QUERY_SQ); 667 MLX5_COMMAND_STR_CASE(CREATE_RQ); 668 MLX5_COMMAND_STR_CASE(MODIFY_RQ); 669 MLX5_COMMAND_STR_CASE(DESTROY_RQ); 670 MLX5_COMMAND_STR_CASE(QUERY_RQ); 671 MLX5_COMMAND_STR_CASE(CREATE_RMP); 672 MLX5_COMMAND_STR_CASE(MODIFY_RMP); 673 MLX5_COMMAND_STR_CASE(DESTROY_RMP); 674 MLX5_COMMAND_STR_CASE(QUERY_RMP); 675 MLX5_COMMAND_STR_CASE(CREATE_TIS); 676 MLX5_COMMAND_STR_CASE(MODIFY_TIS); 677 MLX5_COMMAND_STR_CASE(DESTROY_TIS); 678 MLX5_COMMAND_STR_CASE(QUERY_TIS); 679 MLX5_COMMAND_STR_CASE(CREATE_RQT); 680 MLX5_COMMAND_STR_CASE(MODIFY_RQT); 681 MLX5_COMMAND_STR_CASE(DESTROY_RQT); 682 MLX5_COMMAND_STR_CASE(QUERY_RQT); 683 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT); 684 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE); 685 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE); 686 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE); 687 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP); 688 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP); 689 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP); 690 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY); 691 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY); 692 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY); 693 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER); 694 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER); 695 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER); 696 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE); 697 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT); 698 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT); 699 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT); 700 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT); 701 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP); 702 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP); 703 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP); 704 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS); 705 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP); 706 MLX5_COMMAND_STR_CASE(CREATE_XRQ); 707 MLX5_COMMAND_STR_CASE(DESTROY_XRQ); 708 MLX5_COMMAND_STR_CASE(QUERY_XRQ); 709 MLX5_COMMAND_STR_CASE(ARM_XRQ); 710 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT); 711 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT); 712 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT); 713 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT); 714 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT); 715 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC); 716 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC); 717 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS); 718 MLX5_COMMAND_STR_CASE(CREATE_UCTX); 719 MLX5_COMMAND_STR_CASE(DESTROY_UCTX); 720 MLX5_COMMAND_STR_CASE(CREATE_UMEM); 721 MLX5_COMMAND_STR_CASE(DESTROY_UMEM); 722 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR); 723 MLX5_COMMAND_STR_CASE(MODIFY_XRQ); 724 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE); 725 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE); 726 MLX5_COMMAND_STR_CASE(ALLOC_SF); 727 MLX5_COMMAND_STR_CASE(DEALLOC_SF); 728 MLX5_COMMAND_STR_CASE(SUSPEND_VHCA); 729 MLX5_COMMAND_STR_CASE(RESUME_VHCA); 730 MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE); 731 MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE); 732 MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE); 733 MLX5_COMMAND_STR_CASE(SYNC_CRYPTO); 734 default: return "unknown command opcode"; 735 } 736 } 737 738 static const char *cmd_status_str(u8 status) 739 { 740 switch (status) { 741 case MLX5_CMD_STAT_OK: 742 return "OK"; 743 case MLX5_CMD_STAT_INT_ERR: 744 return "internal error"; 745 case MLX5_CMD_STAT_BAD_OP_ERR: 746 return "bad operation"; 747 case MLX5_CMD_STAT_BAD_PARAM_ERR: 748 return "bad parameter"; 749 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: 750 return "bad system state"; 751 case MLX5_CMD_STAT_BAD_RES_ERR: 752 return "bad resource"; 753 case MLX5_CMD_STAT_RES_BUSY: 754 return "resource busy"; 755 case MLX5_CMD_STAT_LIM_ERR: 756 return "limits exceeded"; 757 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: 758 return "bad resource state"; 759 case MLX5_CMD_STAT_IX_ERR: 760 return "bad index"; 761 case MLX5_CMD_STAT_NO_RES_ERR: 762 return "no resources"; 763 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: 764 return "bad input length"; 765 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: 766 return "bad output length"; 767 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: 768 return "bad QP state"; 769 case MLX5_CMD_STAT_BAD_PKT_ERR: 770 return "bad packet (discarded)"; 771 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: 772 return "bad size too many outstanding CQEs"; 773 default: 774 return "unknown status"; 775 } 776 } 777 778 static int cmd_status_to_err(u8 status) 779 { 780 switch (status) { 781 case MLX5_CMD_STAT_OK: return 0; 782 case MLX5_CMD_STAT_INT_ERR: return -EIO; 783 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL; 784 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL; 785 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO; 786 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL; 787 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY; 788 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM; 789 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL; 790 case MLX5_CMD_STAT_IX_ERR: return -EINVAL; 791 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN; 792 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO; 793 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO; 794 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL; 795 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL; 796 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL; 797 default: return -EIO; 798 } 799 } 800 801 void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out) 802 { 803 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 804 u8 status = MLX5_GET(mbox_out, out, status); 805 806 mlx5_core_err_rl(dev, 807 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n", 808 mlx5_command_str(opcode), opcode, op_mod, 809 cmd_status_str(status), status, syndrome, cmd_status_to_err(status)); 810 } 811 EXPORT_SYMBOL(mlx5_cmd_out_err); 812 813 static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out) 814 { 815 u16 opcode, op_mod; 816 u16 uid; 817 818 opcode = in_to_opcode(in); 819 op_mod = MLX5_GET(mbox_in, in, op_mod); 820 uid = MLX5_GET(mbox_in, in, uid); 821 822 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY && 823 opcode != MLX5_CMD_OP_CREATE_UCTX) 824 mlx5_cmd_out_err(dev, opcode, op_mod, out); 825 } 826 827 int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out) 828 { 829 /* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */ 830 if (err == -ENXIO) { 831 u16 opcode = in_to_opcode(in); 832 u32 syndrome; 833 u8 status; 834 835 /* PCI Error, emulate command return status, for smooth reset */ 836 err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status); 837 MLX5_SET(mbox_out, out, status, status); 838 MLX5_SET(mbox_out, out, syndrome, syndrome); 839 if (!err) 840 return 0; 841 } 842 843 /* driver or FW delivery error */ 844 if (err != -EREMOTEIO && err) 845 return err; 846 847 /* check outbox status */ 848 err = cmd_status_to_err(MLX5_GET(mbox_out, out, status)); 849 if (err) 850 cmd_status_print(dev, in, out); 851 852 return err; 853 } 854 EXPORT_SYMBOL(mlx5_cmd_check); 855 856 static void dump_command(struct mlx5_core_dev *dev, 857 struct mlx5_cmd_work_ent *ent, int input) 858 { 859 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out; 860 struct mlx5_cmd_mailbox *next = msg->next; 861 int n = mlx5_calc_cmd_blocks(msg); 862 u16 op = ent->op; 863 int data_only; 864 u32 offset = 0; 865 int dump_len; 866 int i; 867 868 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx); 869 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA)); 870 871 if (data_only) 872 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA, 873 "cmd[%d]: dump command data %s(0x%x) %s\n", 874 ent->idx, mlx5_command_str(op), op, 875 input ? "INPUT" : "OUTPUT"); 876 else 877 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n", 878 ent->idx, mlx5_command_str(op), op, 879 input ? "INPUT" : "OUTPUT"); 880 881 if (data_only) { 882 if (input) { 883 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx); 884 offset += sizeof(ent->lay->in); 885 } else { 886 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx); 887 offset += sizeof(ent->lay->out); 888 } 889 } else { 890 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx); 891 offset += sizeof(*ent->lay); 892 } 893 894 for (i = 0; i < n && next; i++) { 895 if (data_only) { 896 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset); 897 dump_buf(next->buf, dump_len, 1, offset, ent->idx); 898 offset += MLX5_CMD_DATA_BLOCK_SIZE; 899 } else { 900 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx); 901 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset, 902 ent->idx); 903 offset += sizeof(struct mlx5_cmd_prot_block); 904 } 905 next = next->next; 906 } 907 908 if (data_only) 909 pr_debug("\n"); 910 911 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx); 912 } 913 914 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced); 915 916 static void cb_timeout_handler(struct work_struct *work) 917 { 918 struct delayed_work *dwork = container_of(work, struct delayed_work, 919 work); 920 struct mlx5_cmd_work_ent *ent = container_of(dwork, 921 struct mlx5_cmd_work_ent, 922 cb_timeout_work); 923 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, 924 cmd); 925 926 mlx5_cmd_eq_recover(dev); 927 928 /* Maybe got handled by eq recover ? */ 929 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) { 930 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx, 931 mlx5_command_str(ent->op), ent->op); 932 goto out; /* phew, already handled */ 933 } 934 935 ent->ret = -ETIMEDOUT; 936 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n", 937 ent->idx, mlx5_command_str(ent->op), ent->op); 938 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 939 940 out: 941 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */ 942 } 943 944 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg); 945 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, 946 struct mlx5_cmd_msg *msg); 947 948 static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode) 949 { 950 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL) 951 return true; 952 953 return cmd->allowed_opcode == opcode; 954 } 955 956 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev) 957 { 958 return pci_channel_offline(dev->pdev) || 959 dev->cmd.state != MLX5_CMDIF_STATE_UP || 960 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR; 961 } 962 963 static void cmd_work_handler(struct work_struct *work) 964 { 965 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work); 966 struct mlx5_cmd *cmd = ent->cmd; 967 bool poll_cmd = ent->polling; 968 struct mlx5_cmd_layout *lay; 969 struct mlx5_core_dev *dev; 970 unsigned long cb_timeout; 971 struct semaphore *sem; 972 unsigned long flags; 973 int alloc_ret; 974 int cmd_mode; 975 976 dev = container_of(cmd, struct mlx5_core_dev, cmd); 977 cb_timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD)); 978 979 complete(&ent->handling); 980 sem = ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem; 981 down(sem); 982 if (!ent->page_queue) { 983 alloc_ret = cmd_alloc_index(cmd, ent); 984 if (alloc_ret < 0) { 985 mlx5_core_err_rl(dev, "failed to allocate command entry\n"); 986 if (ent->callback) { 987 ent->callback(-EAGAIN, ent->context); 988 mlx5_free_cmd_msg(dev, ent->out); 989 free_msg(dev, ent->in); 990 cmd_ent_put(ent); 991 } else { 992 ent->ret = -EAGAIN; 993 complete(&ent->done); 994 } 995 up(sem); 996 return; 997 } 998 } else { 999 ent->idx = cmd->vars.max_reg_cmds; 1000 spin_lock_irqsave(&cmd->alloc_lock, flags); 1001 clear_bit(ent->idx, &cmd->vars.bitmask); 1002 cmd->ent_arr[ent->idx] = ent; 1003 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 1004 } 1005 1006 lay = get_inst(cmd, ent->idx); 1007 ent->lay = lay; 1008 memset(lay, 0, sizeof(*lay)); 1009 memcpy(lay->in, ent->in->first.data, sizeof(lay->in)); 1010 if (ent->in->next) 1011 lay->in_ptr = cpu_to_be64(ent->in->next->dma); 1012 lay->inlen = cpu_to_be32(ent->in->len); 1013 if (ent->out->next) 1014 lay->out_ptr = cpu_to_be64(ent->out->next->dma); 1015 lay->outlen = cpu_to_be32(ent->out->len); 1016 lay->type = MLX5_PCI_CMD_XPORT; 1017 lay->token = ent->token; 1018 lay->status_own = CMD_OWNER_HW; 1019 set_signature(ent, !cmd->checksum_disabled); 1020 dump_command(dev, ent, 1); 1021 ent->ts1 = ktime_get_ns(); 1022 cmd_mode = cmd->mode; 1023 1024 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, cb_timeout)) 1025 cmd_ent_get(ent); 1026 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state); 1027 1028 cmd_ent_get(ent); /* for the _real_ FW event on completion */ 1029 /* Skip sending command to fw if internal error */ 1030 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) { 1031 ent->ret = -ENXIO; 1032 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 1033 return; 1034 } 1035 1036 /* ring doorbell after the descriptor is valid */ 1037 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx); 1038 wmb(); 1039 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell); 1040 /* if not in polling don't use ent after this point */ 1041 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) { 1042 poll_timeout(ent); 1043 /* make sure we read the descriptor after ownership is SW */ 1044 rmb(); 1045 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, (ent->ret == -ETIMEDOUT)); 1046 } 1047 } 1048 1049 static int deliv_status_to_err(u8 status) 1050 { 1051 switch (status) { 1052 case MLX5_CMD_DELIVERY_STAT_OK: 1053 case MLX5_DRIVER_STATUS_ABORTED: 1054 return 0; 1055 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: 1056 case MLX5_CMD_DELIVERY_STAT_TOK_ERR: 1057 return -EBADR; 1058 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: 1059 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: 1060 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: 1061 return -EFAULT; /* Bad address */ 1062 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: 1063 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: 1064 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: 1065 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: 1066 return -ENOMSG; 1067 case MLX5_CMD_DELIVERY_STAT_FW_ERR: 1068 return -EIO; 1069 default: 1070 return -EINVAL; 1071 } 1072 } 1073 1074 static const char *deliv_status_to_str(u8 status) 1075 { 1076 switch (status) { 1077 case MLX5_CMD_DELIVERY_STAT_OK: 1078 return "no errors"; 1079 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: 1080 return "signature error"; 1081 case MLX5_CMD_DELIVERY_STAT_TOK_ERR: 1082 return "token error"; 1083 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: 1084 return "bad block number"; 1085 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: 1086 return "output pointer not aligned to block size"; 1087 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: 1088 return "input pointer not aligned to block size"; 1089 case MLX5_CMD_DELIVERY_STAT_FW_ERR: 1090 return "firmware internal error"; 1091 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: 1092 return "command input length error"; 1093 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: 1094 return "command output length error"; 1095 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: 1096 return "reserved fields not cleared"; 1097 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: 1098 return "bad command descriptor type"; 1099 default: 1100 return "unknown status code"; 1101 } 1102 } 1103 1104 enum { 1105 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000, 1106 }; 1107 1108 static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev, 1109 struct mlx5_cmd_work_ent *ent) 1110 { 1111 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC); 1112 1113 mlx5_cmd_eq_recover(dev); 1114 1115 /* Re-wait on the ent->done after executing the recovery flow. If the 1116 * recovery flow (or any other recovery flow running simultaneously) 1117 * has recovered an EQE, it should cause the entry to be completed by 1118 * the command interface. 1119 */ 1120 if (wait_for_completion_timeout(&ent->done, timeout)) { 1121 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx, 1122 mlx5_command_str(ent->op), ent->op); 1123 return; 1124 } 1125 1126 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx, 1127 mlx5_command_str(ent->op), ent->op); 1128 1129 ent->ret = -ETIMEDOUT; 1130 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true); 1131 } 1132 1133 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) 1134 { 1135 unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD)); 1136 struct mlx5_cmd *cmd = &dev->cmd; 1137 int err; 1138 1139 if (!wait_for_completion_timeout(&ent->handling, timeout) && 1140 cancel_work_sync(&ent->work)) { 1141 ent->ret = -ECANCELED; 1142 goto out_err; 1143 } 1144 if (cmd->mode == CMD_MODE_POLLING || ent->polling) 1145 wait_for_completion(&ent->done); 1146 else if (!wait_for_completion_timeout(&ent->done, timeout)) 1147 wait_func_handle_exec_timeout(dev, ent); 1148 1149 out_err: 1150 err = ent->ret; 1151 1152 if (err == -ETIMEDOUT) { 1153 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", 1154 mlx5_command_str(ent->op), ent->op); 1155 } else if (err == -ECANCELED) { 1156 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n", 1157 mlx5_command_str(ent->op), ent->op); 1158 } 1159 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n", 1160 err, deliv_status_to_str(ent->status), ent->status); 1161 1162 return err; 1163 } 1164 1165 /* Notes: 1166 * 1. Callback functions may not sleep 1167 * 2. page queue commands do not support asynchrous completion 1168 * 1169 * return value in case (!callback): 1170 * ret < 0 : Command execution couldn't be submitted by driver 1171 * ret > 0 : Command execution couldn't be performed by firmware 1172 * ret == 0: Command was executed by FW, Caller must check FW outbox status. 1173 * 1174 * return value in case (callback): 1175 * ret < 0 : Command execution couldn't be submitted by driver 1176 * ret == 0: Command will be submitted to FW for execution 1177 * and the callback will be called for further status updates 1178 */ 1179 static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, 1180 struct mlx5_cmd_msg *out, void *uout, int uout_size, 1181 mlx5_cmd_cbk_t callback, 1182 void *context, int page_queue, 1183 u8 token, bool force_polling) 1184 { 1185 struct mlx5_cmd *cmd = &dev->cmd; 1186 struct mlx5_cmd_work_ent *ent; 1187 struct mlx5_cmd_stats *stats; 1188 u8 status = 0; 1189 int err = 0; 1190 s64 ds; 1191 1192 if (callback && page_queue) 1193 return -EINVAL; 1194 1195 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size, 1196 callback, context, page_queue); 1197 if (IS_ERR(ent)) 1198 return PTR_ERR(ent); 1199 1200 /* put for this ent is when consumed, depending on the use case 1201 * 1) (!callback) blocking flow: by caller after wait_func completes 1202 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled 1203 */ 1204 1205 ent->token = token; 1206 ent->polling = force_polling; 1207 1208 init_completion(&ent->handling); 1209 if (!callback) 1210 init_completion(&ent->done); 1211 1212 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler); 1213 INIT_WORK(&ent->work, cmd_work_handler); 1214 if (page_queue) { 1215 cmd_work_handler(&ent->work); 1216 } else if (!queue_work(cmd->wq, &ent->work)) { 1217 mlx5_core_warn(dev, "failed to queue work\n"); 1218 err = -EALREADY; 1219 goto out_free; 1220 } 1221 1222 if (callback) 1223 return 0; /* mlx5_cmd_comp_handler() will put(ent) */ 1224 1225 err = wait_func(dev, ent); 1226 if (err == -ETIMEDOUT || err == -ECANCELED) 1227 goto out_free; 1228 1229 ds = ent->ts2 - ent->ts1; 1230 stats = xa_load(&cmd->stats, ent->op); 1231 if (stats) { 1232 spin_lock_irq(&stats->lock); 1233 stats->sum += ds; 1234 ++stats->n; 1235 spin_unlock_irq(&stats->lock); 1236 } 1237 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME, 1238 "fw exec time for %s is %lld nsec\n", 1239 mlx5_command_str(ent->op), ds); 1240 1241 out_free: 1242 status = ent->status; 1243 cmd_ent_put(ent); 1244 return err ? : status; 1245 } 1246 1247 static ssize_t dbg_write(struct file *filp, const char __user *buf, 1248 size_t count, loff_t *pos) 1249 { 1250 struct mlx5_core_dev *dev = filp->private_data; 1251 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1252 char lbuf[3]; 1253 int err; 1254 1255 if (!dbg->in_msg || !dbg->out_msg) 1256 return -ENOMEM; 1257 1258 if (count < sizeof(lbuf) - 1) 1259 return -EINVAL; 1260 1261 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1)) 1262 return -EFAULT; 1263 1264 lbuf[sizeof(lbuf) - 1] = 0; 1265 1266 if (strcmp(lbuf, "go")) 1267 return -EINVAL; 1268 1269 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen); 1270 1271 return err ? err : count; 1272 } 1273 1274 static const struct file_operations fops = { 1275 .owner = THIS_MODULE, 1276 .open = simple_open, 1277 .write = dbg_write, 1278 }; 1279 1280 static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size, 1281 u8 token) 1282 { 1283 struct mlx5_cmd_prot_block *block; 1284 struct mlx5_cmd_mailbox *next; 1285 int copy; 1286 1287 if (!to || !from) 1288 return -ENOMEM; 1289 1290 copy = min_t(int, size, sizeof(to->first.data)); 1291 memcpy(to->first.data, from, copy); 1292 size -= copy; 1293 from += copy; 1294 1295 next = to->next; 1296 while (size) { 1297 if (!next) { 1298 /* this is a BUG */ 1299 return -ENOMEM; 1300 } 1301 1302 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE); 1303 block = next->buf; 1304 memcpy(block->data, from, copy); 1305 from += copy; 1306 size -= copy; 1307 block->token = token; 1308 next = next->next; 1309 } 1310 1311 return 0; 1312 } 1313 1314 static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size) 1315 { 1316 struct mlx5_cmd_prot_block *block; 1317 struct mlx5_cmd_mailbox *next; 1318 int copy; 1319 1320 if (!to || !from) 1321 return -ENOMEM; 1322 1323 copy = min_t(int, size, sizeof(from->first.data)); 1324 memcpy(to, from->first.data, copy); 1325 size -= copy; 1326 to += copy; 1327 1328 next = from->next; 1329 while (size) { 1330 if (!next) { 1331 /* this is a BUG */ 1332 return -ENOMEM; 1333 } 1334 1335 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE); 1336 block = next->buf; 1337 1338 memcpy(to, block->data, copy); 1339 to += copy; 1340 size -= copy; 1341 next = next->next; 1342 } 1343 1344 return 0; 1345 } 1346 1347 static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev, 1348 gfp_t flags) 1349 { 1350 struct mlx5_cmd_mailbox *mailbox; 1351 1352 mailbox = kmalloc(sizeof(*mailbox), flags); 1353 if (!mailbox) 1354 return ERR_PTR(-ENOMEM); 1355 1356 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags, 1357 &mailbox->dma); 1358 if (!mailbox->buf) { 1359 mlx5_core_dbg(dev, "failed allocation\n"); 1360 kfree(mailbox); 1361 return ERR_PTR(-ENOMEM); 1362 } 1363 mailbox->next = NULL; 1364 1365 return mailbox; 1366 } 1367 1368 static void free_cmd_box(struct mlx5_core_dev *dev, 1369 struct mlx5_cmd_mailbox *mailbox) 1370 { 1371 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma); 1372 kfree(mailbox); 1373 } 1374 1375 static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev, 1376 gfp_t flags, int size, 1377 u8 token) 1378 { 1379 struct mlx5_cmd_mailbox *tmp, *head = NULL; 1380 struct mlx5_cmd_prot_block *block; 1381 struct mlx5_cmd_msg *msg; 1382 int err; 1383 int n; 1384 int i; 1385 1386 msg = kzalloc(sizeof(*msg), flags); 1387 if (!msg) 1388 return ERR_PTR(-ENOMEM); 1389 1390 msg->len = size; 1391 n = mlx5_calc_cmd_blocks(msg); 1392 1393 for (i = 0; i < n; i++) { 1394 tmp = alloc_cmd_box(dev, flags); 1395 if (IS_ERR(tmp)) { 1396 mlx5_core_warn(dev, "failed allocating block\n"); 1397 err = PTR_ERR(tmp); 1398 goto err_alloc; 1399 } 1400 1401 block = tmp->buf; 1402 tmp->next = head; 1403 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0); 1404 block->block_num = cpu_to_be32(n - i - 1); 1405 block->token = token; 1406 head = tmp; 1407 } 1408 msg->next = head; 1409 return msg; 1410 1411 err_alloc: 1412 while (head) { 1413 tmp = head->next; 1414 free_cmd_box(dev, head); 1415 head = tmp; 1416 } 1417 kfree(msg); 1418 1419 return ERR_PTR(err); 1420 } 1421 1422 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, 1423 struct mlx5_cmd_msg *msg) 1424 { 1425 struct mlx5_cmd_mailbox *head = msg->next; 1426 struct mlx5_cmd_mailbox *next; 1427 1428 while (head) { 1429 next = head->next; 1430 free_cmd_box(dev, head); 1431 head = next; 1432 } 1433 kfree(msg); 1434 } 1435 1436 static ssize_t data_write(struct file *filp, const char __user *buf, 1437 size_t count, loff_t *pos) 1438 { 1439 struct mlx5_core_dev *dev = filp->private_data; 1440 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1441 void *ptr; 1442 1443 if (*pos != 0) 1444 return -EINVAL; 1445 1446 kfree(dbg->in_msg); 1447 dbg->in_msg = NULL; 1448 dbg->inlen = 0; 1449 ptr = memdup_user(buf, count); 1450 if (IS_ERR(ptr)) 1451 return PTR_ERR(ptr); 1452 dbg->in_msg = ptr; 1453 dbg->inlen = count; 1454 1455 *pos = count; 1456 1457 return count; 1458 } 1459 1460 static ssize_t data_read(struct file *filp, char __user *buf, size_t count, 1461 loff_t *pos) 1462 { 1463 struct mlx5_core_dev *dev = filp->private_data; 1464 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1465 1466 if (!dbg->out_msg) 1467 return -ENOMEM; 1468 1469 return simple_read_from_buffer(buf, count, pos, dbg->out_msg, 1470 dbg->outlen); 1471 } 1472 1473 static const struct file_operations dfops = { 1474 .owner = THIS_MODULE, 1475 .open = simple_open, 1476 .write = data_write, 1477 .read = data_read, 1478 }; 1479 1480 static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count, 1481 loff_t *pos) 1482 { 1483 struct mlx5_core_dev *dev = filp->private_data; 1484 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1485 char outlen[8]; 1486 int err; 1487 1488 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen); 1489 if (err < 0) 1490 return err; 1491 1492 return simple_read_from_buffer(buf, count, pos, outlen, err); 1493 } 1494 1495 static ssize_t outlen_write(struct file *filp, const char __user *buf, 1496 size_t count, loff_t *pos) 1497 { 1498 struct mlx5_core_dev *dev = filp->private_data; 1499 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1500 char outlen_str[8] = {0}; 1501 int outlen; 1502 void *ptr; 1503 int err; 1504 1505 if (*pos != 0 || count > 6) 1506 return -EINVAL; 1507 1508 kfree(dbg->out_msg); 1509 dbg->out_msg = NULL; 1510 dbg->outlen = 0; 1511 1512 if (copy_from_user(outlen_str, buf, count)) 1513 return -EFAULT; 1514 1515 err = sscanf(outlen_str, "%d", &outlen); 1516 if (err != 1) 1517 return -EINVAL; 1518 1519 ptr = kzalloc(outlen, GFP_KERNEL); 1520 if (!ptr) 1521 return -ENOMEM; 1522 1523 dbg->out_msg = ptr; 1524 dbg->outlen = outlen; 1525 1526 *pos = count; 1527 1528 return count; 1529 } 1530 1531 static const struct file_operations olfops = { 1532 .owner = THIS_MODULE, 1533 .open = simple_open, 1534 .write = outlen_write, 1535 .read = outlen_read, 1536 }; 1537 1538 static void set_wqname(struct mlx5_core_dev *dev) 1539 { 1540 struct mlx5_cmd *cmd = &dev->cmd; 1541 1542 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s", 1543 dev_name(dev->device)); 1544 } 1545 1546 static void clean_debug_files(struct mlx5_core_dev *dev) 1547 { 1548 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1549 1550 if (!mlx5_debugfs_root) 1551 return; 1552 1553 debugfs_remove_recursive(dbg->dbg_root); 1554 } 1555 1556 static void create_debugfs_files(struct mlx5_core_dev *dev) 1557 { 1558 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1559 1560 dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev)); 1561 1562 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops); 1563 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops); 1564 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops); 1565 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status); 1566 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops); 1567 } 1568 1569 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode) 1570 { 1571 struct mlx5_cmd *cmd = &dev->cmd; 1572 int i; 1573 1574 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1575 down(&cmd->vars.sem); 1576 down(&cmd->vars.pages_sem); 1577 1578 cmd->allowed_opcode = opcode; 1579 1580 up(&cmd->vars.pages_sem); 1581 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1582 up(&cmd->vars.sem); 1583 } 1584 1585 static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) 1586 { 1587 struct mlx5_cmd *cmd = &dev->cmd; 1588 int i; 1589 1590 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1591 down(&cmd->vars.sem); 1592 down(&cmd->vars.pages_sem); 1593 1594 cmd->mode = mode; 1595 1596 up(&cmd->vars.pages_sem); 1597 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1598 up(&cmd->vars.sem); 1599 } 1600 1601 static int cmd_comp_notifier(struct notifier_block *nb, 1602 unsigned long type, void *data) 1603 { 1604 struct mlx5_core_dev *dev; 1605 struct mlx5_cmd *cmd; 1606 struct mlx5_eqe *eqe; 1607 1608 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb); 1609 dev = container_of(cmd, struct mlx5_core_dev, cmd); 1610 eqe = data; 1611 1612 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false); 1613 1614 return NOTIFY_OK; 1615 } 1616 void mlx5_cmd_use_events(struct mlx5_core_dev *dev) 1617 { 1618 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD); 1619 mlx5_eq_notifier_register(dev, &dev->cmd.nb); 1620 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS); 1621 } 1622 1623 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev) 1624 { 1625 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING); 1626 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb); 1627 } 1628 1629 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg) 1630 { 1631 unsigned long flags; 1632 1633 if (msg->parent) { 1634 spin_lock_irqsave(&msg->parent->lock, flags); 1635 list_add_tail(&msg->list, &msg->parent->head); 1636 spin_unlock_irqrestore(&msg->parent->lock, flags); 1637 } else { 1638 mlx5_free_cmd_msg(dev, msg); 1639 } 1640 } 1641 1642 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced) 1643 { 1644 struct mlx5_cmd *cmd = &dev->cmd; 1645 struct mlx5_cmd_work_ent *ent; 1646 mlx5_cmd_cbk_t callback; 1647 void *context; 1648 int err; 1649 int i; 1650 s64 ds; 1651 struct mlx5_cmd_stats *stats; 1652 unsigned long flags; 1653 unsigned long vector; 1654 1655 /* there can be at most 32 command queues */ 1656 vector = vec & 0xffffffff; 1657 for (i = 0; i < (1 << cmd->vars.log_sz); i++) { 1658 if (test_bit(i, &vector)) { 1659 ent = cmd->ent_arr[i]; 1660 1661 /* if we already completed the command, ignore it */ 1662 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, 1663 &ent->state)) { 1664 /* only real completion can free the cmd slot */ 1665 if (!forced) { 1666 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n", 1667 ent->idx); 1668 cmd_ent_put(ent); 1669 } 1670 continue; 1671 } 1672 1673 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work)) 1674 cmd_ent_put(ent); /* timeout work was canceled */ 1675 1676 if (!forced || /* Real FW completion */ 1677 mlx5_cmd_is_down(dev) || /* No real FW completion is expected */ 1678 !opcode_allowed(cmd, ent->op)) 1679 cmd_ent_put(ent); 1680 1681 ent->ts2 = ktime_get_ns(); 1682 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out)); 1683 dump_command(dev, ent, 0); 1684 1685 if (vec & MLX5_TRIGGERED_CMD_COMP) 1686 ent->ret = -ENXIO; 1687 1688 if (!ent->ret) { /* Command completed by FW */ 1689 if (!cmd->checksum_disabled) 1690 ent->ret = verify_signature(ent); 1691 1692 ent->status = ent->lay->status_own >> 1; 1693 1694 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n", 1695 ent->ret, deliv_status_to_str(ent->status), ent->status); 1696 } 1697 1698 if (ent->callback) { 1699 ds = ent->ts2 - ent->ts1; 1700 stats = xa_load(&cmd->stats, ent->op); 1701 if (stats) { 1702 spin_lock_irqsave(&stats->lock, flags); 1703 stats->sum += ds; 1704 ++stats->n; 1705 spin_unlock_irqrestore(&stats->lock, flags); 1706 } 1707 1708 callback = ent->callback; 1709 context = ent->context; 1710 err = ent->ret ? : ent->status; 1711 if (err > 0) /* Failed in FW, command didn't execute */ 1712 err = deliv_status_to_err(err); 1713 1714 if (!err) 1715 err = mlx5_copy_from_msg(ent->uout, 1716 ent->out, 1717 ent->uout_size); 1718 1719 mlx5_free_cmd_msg(dev, ent->out); 1720 free_msg(dev, ent->in); 1721 1722 /* final consumer is done, release ent */ 1723 cmd_ent_put(ent); 1724 callback(err, context); 1725 } else { 1726 /* release wait_func() so mlx5_cmd_invoke() 1727 * can make the final ent_put() 1728 */ 1729 complete(&ent->done); 1730 } 1731 } 1732 } 1733 } 1734 1735 static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev) 1736 { 1737 struct mlx5_cmd *cmd = &dev->cmd; 1738 unsigned long bitmask; 1739 unsigned long flags; 1740 u64 vector; 1741 int i; 1742 1743 /* wait for pending handlers to complete */ 1744 mlx5_eq_synchronize_cmd_irq(dev); 1745 spin_lock_irqsave(&dev->cmd.alloc_lock, flags); 1746 vector = ~dev->cmd.vars.bitmask & ((1ul << (1 << dev->cmd.vars.log_sz)) - 1); 1747 if (!vector) 1748 goto no_trig; 1749 1750 bitmask = vector; 1751 /* we must increment the allocated entries refcount before triggering the completions 1752 * to guarantee pending commands will not get freed in the meanwhile. 1753 * For that reason, it also has to be done inside the alloc_lock. 1754 */ 1755 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz)) 1756 cmd_ent_get(cmd->ent_arr[i]); 1757 vector |= MLX5_TRIGGERED_CMD_COMP; 1758 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 1759 1760 mlx5_core_dbg(dev, "vector 0x%llx\n", vector); 1761 mlx5_cmd_comp_handler(dev, vector, true); 1762 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz)) 1763 cmd_ent_put(cmd->ent_arr[i]); 1764 return; 1765 1766 no_trig: 1767 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 1768 } 1769 1770 void mlx5_cmd_flush(struct mlx5_core_dev *dev) 1771 { 1772 struct mlx5_cmd *cmd = &dev->cmd; 1773 int i; 1774 1775 for (i = 0; i < cmd->vars.max_reg_cmds; i++) { 1776 while (down_trylock(&cmd->vars.sem)) { 1777 mlx5_cmd_trigger_completions(dev); 1778 cond_resched(); 1779 } 1780 } 1781 1782 while (down_trylock(&cmd->vars.pages_sem)) { 1783 mlx5_cmd_trigger_completions(dev); 1784 cond_resched(); 1785 } 1786 1787 /* Unlock cmdif */ 1788 up(&cmd->vars.pages_sem); 1789 for (i = 0; i < cmd->vars.max_reg_cmds; i++) 1790 up(&cmd->vars.sem); 1791 } 1792 1793 static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size, 1794 gfp_t gfp) 1795 { 1796 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM); 1797 struct cmd_msg_cache *ch = NULL; 1798 struct mlx5_cmd *cmd = &dev->cmd; 1799 int i; 1800 1801 if (in_size <= 16) 1802 goto cache_miss; 1803 1804 for (i = 0; i < dev->profile.num_cmd_caches; i++) { 1805 ch = &cmd->cache[i]; 1806 if (in_size > ch->max_inbox_size) 1807 continue; 1808 spin_lock_irq(&ch->lock); 1809 if (list_empty(&ch->head)) { 1810 spin_unlock_irq(&ch->lock); 1811 continue; 1812 } 1813 msg = list_entry(ch->head.next, typeof(*msg), list); 1814 /* For cached lists, we must explicitly state what is 1815 * the real size 1816 */ 1817 msg->len = in_size; 1818 list_del(&msg->list); 1819 spin_unlock_irq(&ch->lock); 1820 break; 1821 } 1822 1823 if (!IS_ERR(msg)) 1824 return msg; 1825 1826 cache_miss: 1827 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0); 1828 return msg; 1829 } 1830 1831 static int is_manage_pages(void *in) 1832 { 1833 return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES; 1834 } 1835 1836 /* Notes: 1837 * 1. Callback functions may not sleep 1838 * 2. Page queue commands do not support asynchrous completion 1839 */ 1840 static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 1841 int out_size, mlx5_cmd_cbk_t callback, void *context, 1842 bool force_polling) 1843 { 1844 struct mlx5_cmd_msg *inb, *outb; 1845 u16 opcode = in_to_opcode(in); 1846 bool throttle_op; 1847 int pages_queue; 1848 gfp_t gfp; 1849 u8 token; 1850 int err; 1851 1852 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode)) 1853 return -ENXIO; 1854 1855 throttle_op = mlx5_cmd_is_throttle_opcode(opcode); 1856 if (throttle_op) { 1857 /* atomic context may not sleep */ 1858 if (callback) 1859 return -EINVAL; 1860 down(&dev->cmd.vars.throttle_sem); 1861 } 1862 1863 pages_queue = is_manage_pages(in); 1864 gfp = callback ? GFP_ATOMIC : GFP_KERNEL; 1865 1866 inb = alloc_msg(dev, in_size, gfp); 1867 if (IS_ERR(inb)) { 1868 err = PTR_ERR(inb); 1869 goto out_up; 1870 } 1871 1872 token = alloc_token(&dev->cmd); 1873 1874 err = mlx5_copy_to_msg(inb, in, in_size, token); 1875 if (err) { 1876 mlx5_core_warn(dev, "err %d\n", err); 1877 goto out_in; 1878 } 1879 1880 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token); 1881 if (IS_ERR(outb)) { 1882 err = PTR_ERR(outb); 1883 goto out_in; 1884 } 1885 1886 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context, 1887 pages_queue, token, force_polling); 1888 if (callback) 1889 return err; 1890 1891 if (err > 0) /* Failed in FW, command didn't execute */ 1892 err = deliv_status_to_err(err); 1893 1894 if (err) 1895 goto out_out; 1896 1897 /* command completed by FW */ 1898 err = mlx5_copy_from_msg(out, outb, out_size); 1899 out_out: 1900 mlx5_free_cmd_msg(dev, outb); 1901 out_in: 1902 free_msg(dev, inb); 1903 out_up: 1904 if (throttle_op) 1905 up(&dev->cmd.vars.throttle_sem); 1906 return err; 1907 } 1908 1909 static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out) 1910 { 1911 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 1912 u8 status = MLX5_GET(mbox_out, out, status); 1913 1914 trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod, 1915 cmd_status_str(status), status, syndrome, 1916 cmd_status_to_err(status)); 1917 } 1918 1919 static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status, 1920 u32 syndrome, int err) 1921 { 1922 const char *namep = mlx5_command_str(opcode); 1923 struct mlx5_cmd_stats *stats; 1924 unsigned long flags; 1925 1926 if (!err || !(strcmp(namep, "unknown command opcode"))) 1927 return; 1928 1929 stats = xa_load(&dev->cmd.stats, opcode); 1930 if (!stats) 1931 return; 1932 spin_lock_irqsave(&stats->lock, flags); 1933 stats->failed++; 1934 if (err < 0) 1935 stats->last_failed_errno = -err; 1936 if (err == -EREMOTEIO) { 1937 stats->failed_mbox_status++; 1938 stats->last_failed_mbox_status = status; 1939 stats->last_failed_syndrome = syndrome; 1940 } 1941 spin_unlock_irqrestore(&stats->lock, flags); 1942 } 1943 1944 /* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */ 1945 static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out) 1946 { 1947 u32 syndrome = MLX5_GET(mbox_out, out, syndrome); 1948 u8 status = MLX5_GET(mbox_out, out, status); 1949 1950 if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */ 1951 err = -EIO; 1952 1953 if (!err && status != MLX5_CMD_STAT_OK) { 1954 err = -EREMOTEIO; 1955 mlx5_cmd_err_trace(dev, opcode, op_mod, out); 1956 } 1957 1958 cmd_status_log(dev, opcode, status, syndrome, err); 1959 return err; 1960 } 1961 1962 /** 1963 * mlx5_cmd_do - Executes a fw command, wait for completion. 1964 * Unlike mlx5_cmd_exec, this function will not translate or intercept 1965 * outbox.status and will return -EREMOTEIO when 1966 * outbox.status != MLX5_CMD_STAT_OK 1967 * 1968 * @dev: mlx5 core device 1969 * @in: inbox mlx5_ifc command buffer 1970 * @in_size: inbox buffer size 1971 * @out: outbox mlx5_ifc buffer 1972 * @out_size: outbox size 1973 * 1974 * @return: 1975 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK. 1976 * Caller must check FW outbox status. 1977 * 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK. 1978 * < 0 : Command execution couldn't be performed by firmware or driver 1979 */ 1980 int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size) 1981 { 1982 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false); 1983 u16 op_mod = MLX5_GET(mbox_in, in, op_mod); 1984 u16 opcode = in_to_opcode(in); 1985 1986 return cmd_status_err(dev, err, opcode, op_mod, out); 1987 } 1988 EXPORT_SYMBOL(mlx5_cmd_do); 1989 1990 /** 1991 * mlx5_cmd_exec - Executes a fw command, wait for completion 1992 * 1993 * @dev: mlx5 core device 1994 * @in: inbox mlx5_ifc command buffer 1995 * @in_size: inbox buffer size 1996 * @out: outbox mlx5_ifc buffer 1997 * @out_size: outbox size 1998 * 1999 * @return: 0 if no error, FW command execution was successful 2000 * and outbox status is ok. 2001 */ 2002 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 2003 int out_size) 2004 { 2005 int err = mlx5_cmd_do(dev, in, in_size, out, out_size); 2006 2007 return mlx5_cmd_check(dev, err, in, out); 2008 } 2009 EXPORT_SYMBOL(mlx5_cmd_exec); 2010 2011 /** 2012 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion 2013 * Needed for driver force teardown, when command completion EQ 2014 * will not be available to complete the command 2015 * 2016 * @dev: mlx5 core device 2017 * @in: inbox mlx5_ifc command buffer 2018 * @in_size: inbox buffer size 2019 * @out: outbox mlx5_ifc buffer 2020 * @out_size: outbox size 2021 * 2022 * @return: 0 if no error, FW command execution was successful 2023 * and outbox status is ok. 2024 */ 2025 int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, 2026 void *out, int out_size) 2027 { 2028 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true); 2029 u16 op_mod = MLX5_GET(mbox_in, in, op_mod); 2030 u16 opcode = in_to_opcode(in); 2031 2032 err = cmd_status_err(dev, err, opcode, op_mod, out); 2033 return mlx5_cmd_check(dev, err, in, out); 2034 } 2035 EXPORT_SYMBOL(mlx5_cmd_exec_polling); 2036 2037 void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev, 2038 struct mlx5_async_ctx *ctx) 2039 { 2040 ctx->dev = dev; 2041 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */ 2042 atomic_set(&ctx->num_inflight, 1); 2043 init_completion(&ctx->inflight_done); 2044 } 2045 EXPORT_SYMBOL(mlx5_cmd_init_async_ctx); 2046 2047 /** 2048 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx 2049 * @ctx: The ctx to clean 2050 * 2051 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The 2052 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after 2053 * the call mlx5_cleanup_async_ctx(). 2054 */ 2055 void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx) 2056 { 2057 if (!atomic_dec_and_test(&ctx->num_inflight)) 2058 wait_for_completion(&ctx->inflight_done); 2059 } 2060 EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx); 2061 2062 static void mlx5_cmd_exec_cb_handler(int status, void *_work) 2063 { 2064 struct mlx5_async_work *work = _work; 2065 struct mlx5_async_ctx *ctx; 2066 2067 ctx = work->ctx; 2068 status = cmd_status_err(ctx->dev, status, work->opcode, work->op_mod, work->out); 2069 work->user_callback(status, work); 2070 if (atomic_dec_and_test(&ctx->num_inflight)) 2071 complete(&ctx->inflight_done); 2072 } 2073 2074 int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, 2075 void *out, int out_size, mlx5_async_cbk_t callback, 2076 struct mlx5_async_work *work) 2077 { 2078 int ret; 2079 2080 work->ctx = ctx; 2081 work->user_callback = callback; 2082 work->opcode = in_to_opcode(in); 2083 work->op_mod = MLX5_GET(mbox_in, in, op_mod); 2084 work->out = out; 2085 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight))) 2086 return -EIO; 2087 ret = cmd_exec(ctx->dev, in, in_size, out, out_size, 2088 mlx5_cmd_exec_cb_handler, work, false); 2089 if (ret && atomic_dec_and_test(&ctx->num_inflight)) 2090 complete(&ctx->inflight_done); 2091 2092 return ret; 2093 } 2094 EXPORT_SYMBOL(mlx5_cmd_exec_cb); 2095 2096 static void destroy_msg_cache(struct mlx5_core_dev *dev) 2097 { 2098 struct cmd_msg_cache *ch; 2099 struct mlx5_cmd_msg *msg; 2100 struct mlx5_cmd_msg *n; 2101 int i; 2102 2103 for (i = 0; i < dev->profile.num_cmd_caches; i++) { 2104 ch = &dev->cmd.cache[i]; 2105 list_for_each_entry_safe(msg, n, &ch->head, list) { 2106 list_del(&msg->list); 2107 mlx5_free_cmd_msg(dev, msg); 2108 } 2109 } 2110 } 2111 2112 static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = { 2113 512, 32, 16, 8, 2 2114 }; 2115 2116 static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = { 2117 16 + MLX5_CMD_DATA_BLOCK_SIZE, 2118 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2, 2119 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16, 2120 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256, 2121 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512, 2122 }; 2123 2124 static void create_msg_cache(struct mlx5_core_dev *dev) 2125 { 2126 struct mlx5_cmd *cmd = &dev->cmd; 2127 struct cmd_msg_cache *ch; 2128 struct mlx5_cmd_msg *msg; 2129 int i; 2130 int k; 2131 2132 /* Initialize and fill the caches with initial entries */ 2133 for (k = 0; k < dev->profile.num_cmd_caches; k++) { 2134 ch = &cmd->cache[k]; 2135 spin_lock_init(&ch->lock); 2136 INIT_LIST_HEAD(&ch->head); 2137 ch->num_ent = cmd_cache_num_ent[k]; 2138 ch->max_inbox_size = cmd_cache_ent_size[k]; 2139 for (i = 0; i < ch->num_ent; i++) { 2140 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN, 2141 ch->max_inbox_size, 0); 2142 if (IS_ERR(msg)) 2143 break; 2144 msg->parent = ch; 2145 list_add_tail(&msg->list, &ch->head); 2146 } 2147 } 2148 } 2149 2150 static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) 2151 { 2152 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, 2153 &cmd->alloc_dma, GFP_KERNEL); 2154 if (!cmd->cmd_alloc_buf) 2155 return -ENOMEM; 2156 2157 /* make sure it is aligned to 4K */ 2158 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) { 2159 cmd->cmd_buf = cmd->cmd_alloc_buf; 2160 cmd->dma = cmd->alloc_dma; 2161 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE; 2162 return 0; 2163 } 2164 2165 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf, 2166 cmd->alloc_dma); 2167 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), 2168 2 * MLX5_ADAPTER_PAGE_SIZE - 1, 2169 &cmd->alloc_dma, GFP_KERNEL); 2170 if (!cmd->cmd_alloc_buf) 2171 return -ENOMEM; 2172 2173 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE); 2174 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE); 2175 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1; 2176 return 0; 2177 } 2178 2179 static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) 2180 { 2181 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf, 2182 cmd->alloc_dma); 2183 } 2184 2185 static u16 cmdif_rev(struct mlx5_core_dev *dev) 2186 { 2187 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; 2188 } 2189 2190 int mlx5_cmd_init(struct mlx5_core_dev *dev) 2191 { 2192 struct mlx5_cmd *cmd = &dev->cmd; 2193 2194 cmd->checksum_disabled = 1; 2195 2196 spin_lock_init(&cmd->alloc_lock); 2197 spin_lock_init(&cmd->token_lock); 2198 2199 set_wqname(dev); 2200 cmd->wq = create_singlethread_workqueue(cmd->wq_name); 2201 if (!cmd->wq) { 2202 mlx5_core_err(dev, "failed to create command workqueue\n"); 2203 return -ENOMEM; 2204 } 2205 2206 mlx5_cmdif_debugfs_init(dev); 2207 2208 return 0; 2209 } 2210 2211 void mlx5_cmd_cleanup(struct mlx5_core_dev *dev) 2212 { 2213 struct mlx5_cmd *cmd = &dev->cmd; 2214 2215 mlx5_cmdif_debugfs_cleanup(dev); 2216 destroy_workqueue(cmd->wq); 2217 } 2218 2219 int mlx5_cmd_enable(struct mlx5_core_dev *dev) 2220 { 2221 int size = sizeof(struct mlx5_cmd_prot_block); 2222 int align = roundup_pow_of_two(size); 2223 struct mlx5_cmd *cmd = &dev->cmd; 2224 u32 cmd_h, cmd_l; 2225 int err; 2226 2227 memset(&cmd->vars, 0, sizeof(cmd->vars)); 2228 cmd->vars.cmdif_rev = cmdif_rev(dev); 2229 if (cmd->vars.cmdif_rev != CMD_IF_REV) { 2230 mlx5_core_err(dev, 2231 "Driver cmdif rev(%d) differs from firmware's(%d)\n", 2232 CMD_IF_REV, cmd->vars.cmdif_rev); 2233 return -EINVAL; 2234 } 2235 2236 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff; 2237 cmd->vars.log_sz = cmd_l >> 4 & 0xf; 2238 cmd->vars.log_stride = cmd_l & 0xf; 2239 if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) { 2240 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n", 2241 1 << cmd->vars.log_sz); 2242 return -EINVAL; 2243 } 2244 2245 if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) { 2246 mlx5_core_err(dev, "command queue size overflow\n"); 2247 return -EINVAL; 2248 } 2249 2250 cmd->state = MLX5_CMDIF_STATE_DOWN; 2251 cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1; 2252 cmd->vars.bitmask = (1UL << cmd->vars.max_reg_cmds) - 1; 2253 2254 sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds); 2255 sema_init(&cmd->vars.pages_sem, 1); 2256 sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2)); 2257 2258 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0); 2259 if (!cmd->pool) 2260 return -ENOMEM; 2261 2262 err = alloc_cmd_page(dev, cmd); 2263 if (err) 2264 goto err_free_pool; 2265 2266 cmd_h = (u32)((u64)(cmd->dma) >> 32); 2267 cmd_l = (u32)(cmd->dma); 2268 if (cmd_l & 0xfff) { 2269 mlx5_core_err(dev, "invalid command queue address\n"); 2270 err = -ENOMEM; 2271 goto err_cmd_page; 2272 } 2273 2274 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h); 2275 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz); 2276 2277 /* Make sure firmware sees the complete address before we proceed */ 2278 wmb(); 2279 2280 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); 2281 2282 cmd->mode = CMD_MODE_POLLING; 2283 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL; 2284 2285 create_msg_cache(dev); 2286 create_debugfs_files(dev); 2287 2288 return 0; 2289 2290 err_cmd_page: 2291 free_cmd_page(dev, cmd); 2292 err_free_pool: 2293 dma_pool_destroy(cmd->pool); 2294 return err; 2295 } 2296 2297 void mlx5_cmd_disable(struct mlx5_core_dev *dev) 2298 { 2299 struct mlx5_cmd *cmd = &dev->cmd; 2300 2301 flush_workqueue(cmd->wq); 2302 clean_debug_files(dev); 2303 destroy_msg_cache(dev); 2304 free_cmd_page(dev, cmd); 2305 dma_pool_destroy(cmd->pool); 2306 } 2307 2308 void mlx5_cmd_set_state(struct mlx5_core_dev *dev, 2309 enum mlx5_cmdif_state cmdif_state) 2310 { 2311 dev->cmd.state = cmdif_state; 2312 } 2313