1 /* 2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/highmem.h> 34 #include <linux/module.h> 35 #include <linux/errno.h> 36 #include <linux/pci.h> 37 #include <linux/dma-mapping.h> 38 #include <linux/slab.h> 39 #include <linux/delay.h> 40 #include <linux/random.h> 41 #include <linux/io-mapping.h> 42 #include <linux/mlx5/driver.h> 43 #include <linux/mlx5/eq.h> 44 #include <linux/debugfs.h> 45 46 #include "mlx5_core.h" 47 #include "lib/eq.h" 48 49 enum { 50 CMD_IF_REV = 5, 51 }; 52 53 enum { 54 CMD_MODE_POLLING, 55 CMD_MODE_EVENTS 56 }; 57 58 enum { 59 MLX5_CMD_DELIVERY_STAT_OK = 0x0, 60 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1, 61 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2, 62 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3, 63 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4, 64 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5, 65 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6, 66 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7, 67 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8, 68 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9, 69 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10, 70 }; 71 72 static struct mlx5_cmd_work_ent * 73 cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in, 74 struct mlx5_cmd_msg *out, void *uout, int uout_size, 75 mlx5_cmd_cbk_t cbk, void *context, int page_queue) 76 { 77 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL; 78 struct mlx5_cmd_work_ent *ent; 79 80 ent = kzalloc(sizeof(*ent), alloc_flags); 81 if (!ent) 82 return ERR_PTR(-ENOMEM); 83 84 ent->idx = -EINVAL; 85 ent->in = in; 86 ent->out = out; 87 ent->uout = uout; 88 ent->uout_size = uout_size; 89 ent->callback = cbk; 90 ent->context = context; 91 ent->cmd = cmd; 92 ent->page_queue = page_queue; 93 refcount_set(&ent->refcnt, 1); 94 95 return ent; 96 } 97 98 static void cmd_free_ent(struct mlx5_cmd_work_ent *ent) 99 { 100 kfree(ent); 101 } 102 103 static u8 alloc_token(struct mlx5_cmd *cmd) 104 { 105 u8 token; 106 107 spin_lock(&cmd->token_lock); 108 cmd->token++; 109 if (cmd->token == 0) 110 cmd->token++; 111 token = cmd->token; 112 spin_unlock(&cmd->token_lock); 113 114 return token; 115 } 116 117 static int cmd_alloc_index(struct mlx5_cmd *cmd) 118 { 119 unsigned long flags; 120 int ret; 121 122 spin_lock_irqsave(&cmd->alloc_lock, flags); 123 ret = find_first_bit(&cmd->bitmask, cmd->max_reg_cmds); 124 if (ret < cmd->max_reg_cmds) 125 clear_bit(ret, &cmd->bitmask); 126 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 127 128 return ret < cmd->max_reg_cmds ? ret : -ENOMEM; 129 } 130 131 static void cmd_free_index(struct mlx5_cmd *cmd, int idx) 132 { 133 unsigned long flags; 134 135 spin_lock_irqsave(&cmd->alloc_lock, flags); 136 set_bit(idx, &cmd->bitmask); 137 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 138 } 139 140 static void cmd_ent_get(struct mlx5_cmd_work_ent *ent) 141 { 142 refcount_inc(&ent->refcnt); 143 } 144 145 static void cmd_ent_put(struct mlx5_cmd_work_ent *ent) 146 { 147 if (!refcount_dec_and_test(&ent->refcnt)) 148 return; 149 150 if (ent->idx >= 0) 151 cmd_free_index(ent->cmd, ent->idx); 152 153 cmd_free_ent(ent); 154 } 155 156 static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx) 157 { 158 return cmd->cmd_buf + (idx << cmd->log_stride); 159 } 160 161 static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg) 162 { 163 int size = msg->len; 164 int blen = size - min_t(int, sizeof(msg->first.data), size); 165 166 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE); 167 } 168 169 static u8 xor8_buf(void *buf, size_t offset, int len) 170 { 171 u8 *ptr = buf; 172 u8 sum = 0; 173 int i; 174 int end = len + offset; 175 176 for (i = offset; i < end; i++) 177 sum ^= ptr[i]; 178 179 return sum; 180 } 181 182 static int verify_block_sig(struct mlx5_cmd_prot_block *block) 183 { 184 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0); 185 int xor_len = sizeof(*block) - sizeof(block->data) - 1; 186 187 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff) 188 return -EINVAL; 189 190 if (xor8_buf(block, 0, sizeof(*block)) != 0xff) 191 return -EINVAL; 192 193 return 0; 194 } 195 196 static void calc_block_sig(struct mlx5_cmd_prot_block *block) 197 { 198 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2; 199 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0); 200 201 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len); 202 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1); 203 } 204 205 static void calc_chain_sig(struct mlx5_cmd_msg *msg) 206 { 207 struct mlx5_cmd_mailbox *next = msg->next; 208 int n = mlx5_calc_cmd_blocks(msg); 209 int i = 0; 210 211 for (i = 0; i < n && next; i++) { 212 calc_block_sig(next->buf); 213 next = next->next; 214 } 215 } 216 217 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum) 218 { 219 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay)); 220 if (csum) { 221 calc_chain_sig(ent->in); 222 calc_chain_sig(ent->out); 223 } 224 } 225 226 static void poll_timeout(struct mlx5_cmd_work_ent *ent) 227 { 228 unsigned long poll_end = jiffies + msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC + 1000); 229 u8 own; 230 231 do { 232 own = READ_ONCE(ent->lay->status_own); 233 if (!(own & CMD_OWNER_HW)) { 234 ent->ret = 0; 235 return; 236 } 237 cond_resched(); 238 } while (time_before(jiffies, poll_end)); 239 240 ent->ret = -ETIMEDOUT; 241 } 242 243 static int verify_signature(struct mlx5_cmd_work_ent *ent) 244 { 245 struct mlx5_cmd_mailbox *next = ent->out->next; 246 int n = mlx5_calc_cmd_blocks(ent->out); 247 int err; 248 u8 sig; 249 int i = 0; 250 251 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay)); 252 if (sig != 0xff) 253 return -EINVAL; 254 255 for (i = 0; i < n && next; i++) { 256 err = verify_block_sig(next->buf); 257 if (err) 258 return err; 259 260 next = next->next; 261 } 262 263 return 0; 264 } 265 266 static void dump_buf(void *buf, int size, int data_only, int offset, int idx) 267 { 268 __be32 *p = buf; 269 int i; 270 271 for (i = 0; i < size; i += 16) { 272 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset, 273 be32_to_cpu(p[0]), be32_to_cpu(p[1]), 274 be32_to_cpu(p[2]), be32_to_cpu(p[3])); 275 p += 4; 276 offset += 16; 277 } 278 if (!data_only) 279 pr_debug("\n"); 280 } 281 282 static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, 283 u32 *synd, u8 *status) 284 { 285 *synd = 0; 286 *status = 0; 287 288 switch (op) { 289 case MLX5_CMD_OP_TEARDOWN_HCA: 290 case MLX5_CMD_OP_DISABLE_HCA: 291 case MLX5_CMD_OP_MANAGE_PAGES: 292 case MLX5_CMD_OP_DESTROY_MKEY: 293 case MLX5_CMD_OP_DESTROY_EQ: 294 case MLX5_CMD_OP_DESTROY_CQ: 295 case MLX5_CMD_OP_DESTROY_QP: 296 case MLX5_CMD_OP_DESTROY_PSV: 297 case MLX5_CMD_OP_DESTROY_SRQ: 298 case MLX5_CMD_OP_DESTROY_XRC_SRQ: 299 case MLX5_CMD_OP_DESTROY_XRQ: 300 case MLX5_CMD_OP_DESTROY_DCT: 301 case MLX5_CMD_OP_DEALLOC_Q_COUNTER: 302 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT: 303 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT: 304 case MLX5_CMD_OP_DEALLOC_PD: 305 case MLX5_CMD_OP_DEALLOC_UAR: 306 case MLX5_CMD_OP_DETACH_FROM_MCG: 307 case MLX5_CMD_OP_DEALLOC_XRCD: 308 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN: 309 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT: 310 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY: 311 case MLX5_CMD_OP_DESTROY_LAG: 312 case MLX5_CMD_OP_DESTROY_VPORT_LAG: 313 case MLX5_CMD_OP_DESTROY_TIR: 314 case MLX5_CMD_OP_DESTROY_SQ: 315 case MLX5_CMD_OP_DESTROY_RQ: 316 case MLX5_CMD_OP_DESTROY_RMP: 317 case MLX5_CMD_OP_DESTROY_TIS: 318 case MLX5_CMD_OP_DESTROY_RQT: 319 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 320 case MLX5_CMD_OP_DESTROY_FLOW_GROUP: 321 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY: 322 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER: 323 case MLX5_CMD_OP_2ERR_QP: 324 case MLX5_CMD_OP_2RST_QP: 325 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT: 326 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 327 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 328 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT: 329 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT: 330 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT: 331 case MLX5_CMD_OP_FPGA_DESTROY_QP: 332 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: 333 case MLX5_CMD_OP_DEALLOC_MEMIC: 334 case MLX5_CMD_OP_PAGE_FAULT_RESUME: 335 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS: 336 case MLX5_CMD_OP_DEALLOC_SF: 337 return MLX5_CMD_STAT_OK; 338 339 case MLX5_CMD_OP_QUERY_HCA_CAP: 340 case MLX5_CMD_OP_QUERY_ADAPTER: 341 case MLX5_CMD_OP_INIT_HCA: 342 case MLX5_CMD_OP_ENABLE_HCA: 343 case MLX5_CMD_OP_QUERY_PAGES: 344 case MLX5_CMD_OP_SET_HCA_CAP: 345 case MLX5_CMD_OP_QUERY_ISSI: 346 case MLX5_CMD_OP_SET_ISSI: 347 case MLX5_CMD_OP_CREATE_MKEY: 348 case MLX5_CMD_OP_QUERY_MKEY: 349 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS: 350 case MLX5_CMD_OP_CREATE_EQ: 351 case MLX5_CMD_OP_QUERY_EQ: 352 case MLX5_CMD_OP_GEN_EQE: 353 case MLX5_CMD_OP_CREATE_CQ: 354 case MLX5_CMD_OP_QUERY_CQ: 355 case MLX5_CMD_OP_MODIFY_CQ: 356 case MLX5_CMD_OP_CREATE_QP: 357 case MLX5_CMD_OP_RST2INIT_QP: 358 case MLX5_CMD_OP_INIT2RTR_QP: 359 case MLX5_CMD_OP_RTR2RTS_QP: 360 case MLX5_CMD_OP_RTS2RTS_QP: 361 case MLX5_CMD_OP_SQERR2RTS_QP: 362 case MLX5_CMD_OP_QUERY_QP: 363 case MLX5_CMD_OP_SQD_RTS_QP: 364 case MLX5_CMD_OP_INIT2INIT_QP: 365 case MLX5_CMD_OP_CREATE_PSV: 366 case MLX5_CMD_OP_CREATE_SRQ: 367 case MLX5_CMD_OP_QUERY_SRQ: 368 case MLX5_CMD_OP_ARM_RQ: 369 case MLX5_CMD_OP_CREATE_XRC_SRQ: 370 case MLX5_CMD_OP_QUERY_XRC_SRQ: 371 case MLX5_CMD_OP_ARM_XRC_SRQ: 372 case MLX5_CMD_OP_CREATE_XRQ: 373 case MLX5_CMD_OP_QUERY_XRQ: 374 case MLX5_CMD_OP_ARM_XRQ: 375 case MLX5_CMD_OP_CREATE_DCT: 376 case MLX5_CMD_OP_DRAIN_DCT: 377 case MLX5_CMD_OP_QUERY_DCT: 378 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 379 case MLX5_CMD_OP_QUERY_VPORT_STATE: 380 case MLX5_CMD_OP_MODIFY_VPORT_STATE: 381 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: 382 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT: 383 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT: 384 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS: 385 case MLX5_CMD_OP_SET_ROCE_ADDRESS: 386 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 387 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT: 388 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID: 389 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY: 390 case MLX5_CMD_OP_QUERY_VNIC_ENV: 391 case MLX5_CMD_OP_QUERY_VPORT_COUNTER: 392 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 393 case MLX5_CMD_OP_QUERY_Q_COUNTER: 394 case MLX5_CMD_OP_SET_MONITOR_COUNTER: 395 case MLX5_CMD_OP_ARM_MONITOR_COUNTER: 396 case MLX5_CMD_OP_SET_PP_RATE_LIMIT: 397 case MLX5_CMD_OP_QUERY_RATE_LIMIT: 398 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 399 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 400 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 401 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT: 402 case MLX5_CMD_OP_ALLOC_PD: 403 case MLX5_CMD_OP_ALLOC_UAR: 404 case MLX5_CMD_OP_CONFIG_INT_MODERATION: 405 case MLX5_CMD_OP_ACCESS_REG: 406 case MLX5_CMD_OP_ATTACH_TO_MCG: 407 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG: 408 case MLX5_CMD_OP_MAD_IFC: 409 case MLX5_CMD_OP_QUERY_MAD_DEMUX: 410 case MLX5_CMD_OP_SET_MAD_DEMUX: 411 case MLX5_CMD_OP_NOP: 412 case MLX5_CMD_OP_ALLOC_XRCD: 413 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 414 case MLX5_CMD_OP_QUERY_CONG_STATUS: 415 case MLX5_CMD_OP_MODIFY_CONG_STATUS: 416 case MLX5_CMD_OP_QUERY_CONG_PARAMS: 417 case MLX5_CMD_OP_MODIFY_CONG_PARAMS: 418 case MLX5_CMD_OP_QUERY_CONG_STATISTICS: 419 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 420 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 421 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 422 case MLX5_CMD_OP_CREATE_LAG: 423 case MLX5_CMD_OP_MODIFY_LAG: 424 case MLX5_CMD_OP_QUERY_LAG: 425 case MLX5_CMD_OP_CREATE_VPORT_LAG: 426 case MLX5_CMD_OP_CREATE_TIR: 427 case MLX5_CMD_OP_MODIFY_TIR: 428 case MLX5_CMD_OP_QUERY_TIR: 429 case MLX5_CMD_OP_CREATE_SQ: 430 case MLX5_CMD_OP_MODIFY_SQ: 431 case MLX5_CMD_OP_QUERY_SQ: 432 case MLX5_CMD_OP_CREATE_RQ: 433 case MLX5_CMD_OP_MODIFY_RQ: 434 case MLX5_CMD_OP_QUERY_RQ: 435 case MLX5_CMD_OP_CREATE_RMP: 436 case MLX5_CMD_OP_MODIFY_RMP: 437 case MLX5_CMD_OP_QUERY_RMP: 438 case MLX5_CMD_OP_CREATE_TIS: 439 case MLX5_CMD_OP_MODIFY_TIS: 440 case MLX5_CMD_OP_QUERY_TIS: 441 case MLX5_CMD_OP_CREATE_RQT: 442 case MLX5_CMD_OP_MODIFY_RQT: 443 case MLX5_CMD_OP_QUERY_RQT: 444 445 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 446 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 447 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 448 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 449 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 450 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 451 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 452 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 453 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 454 case MLX5_CMD_OP_FPGA_CREATE_QP: 455 case MLX5_CMD_OP_FPGA_MODIFY_QP: 456 case MLX5_CMD_OP_FPGA_QUERY_QP: 457 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS: 458 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 459 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 460 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 461 case MLX5_CMD_OP_CREATE_UCTX: 462 case MLX5_CMD_OP_DESTROY_UCTX: 463 case MLX5_CMD_OP_CREATE_UMEM: 464 case MLX5_CMD_OP_DESTROY_UMEM: 465 case MLX5_CMD_OP_ALLOC_MEMIC: 466 case MLX5_CMD_OP_MODIFY_XRQ: 467 case MLX5_CMD_OP_RELEASE_XRQ_ERROR: 468 case MLX5_CMD_OP_QUERY_VHCA_STATE: 469 case MLX5_CMD_OP_MODIFY_VHCA_STATE: 470 case MLX5_CMD_OP_ALLOC_SF: 471 *status = MLX5_DRIVER_STATUS_ABORTED; 472 *synd = MLX5_DRIVER_SYND; 473 return -EIO; 474 default: 475 mlx5_core_err(dev, "Unknown FW command (%d)\n", op); 476 return -EINVAL; 477 } 478 } 479 480 const char *mlx5_command_str(int command) 481 { 482 #define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd 483 484 switch (command) { 485 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP); 486 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER); 487 MLX5_COMMAND_STR_CASE(INIT_HCA); 488 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA); 489 MLX5_COMMAND_STR_CASE(ENABLE_HCA); 490 MLX5_COMMAND_STR_CASE(DISABLE_HCA); 491 MLX5_COMMAND_STR_CASE(QUERY_PAGES); 492 MLX5_COMMAND_STR_CASE(MANAGE_PAGES); 493 MLX5_COMMAND_STR_CASE(SET_HCA_CAP); 494 MLX5_COMMAND_STR_CASE(QUERY_ISSI); 495 MLX5_COMMAND_STR_CASE(SET_ISSI); 496 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION); 497 MLX5_COMMAND_STR_CASE(CREATE_MKEY); 498 MLX5_COMMAND_STR_CASE(QUERY_MKEY); 499 MLX5_COMMAND_STR_CASE(DESTROY_MKEY); 500 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS); 501 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME); 502 MLX5_COMMAND_STR_CASE(CREATE_EQ); 503 MLX5_COMMAND_STR_CASE(DESTROY_EQ); 504 MLX5_COMMAND_STR_CASE(QUERY_EQ); 505 MLX5_COMMAND_STR_CASE(GEN_EQE); 506 MLX5_COMMAND_STR_CASE(CREATE_CQ); 507 MLX5_COMMAND_STR_CASE(DESTROY_CQ); 508 MLX5_COMMAND_STR_CASE(QUERY_CQ); 509 MLX5_COMMAND_STR_CASE(MODIFY_CQ); 510 MLX5_COMMAND_STR_CASE(CREATE_QP); 511 MLX5_COMMAND_STR_CASE(DESTROY_QP); 512 MLX5_COMMAND_STR_CASE(RST2INIT_QP); 513 MLX5_COMMAND_STR_CASE(INIT2RTR_QP); 514 MLX5_COMMAND_STR_CASE(RTR2RTS_QP); 515 MLX5_COMMAND_STR_CASE(RTS2RTS_QP); 516 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP); 517 MLX5_COMMAND_STR_CASE(2ERR_QP); 518 MLX5_COMMAND_STR_CASE(2RST_QP); 519 MLX5_COMMAND_STR_CASE(QUERY_QP); 520 MLX5_COMMAND_STR_CASE(SQD_RTS_QP); 521 MLX5_COMMAND_STR_CASE(INIT2INIT_QP); 522 MLX5_COMMAND_STR_CASE(CREATE_PSV); 523 MLX5_COMMAND_STR_CASE(DESTROY_PSV); 524 MLX5_COMMAND_STR_CASE(CREATE_SRQ); 525 MLX5_COMMAND_STR_CASE(DESTROY_SRQ); 526 MLX5_COMMAND_STR_CASE(QUERY_SRQ); 527 MLX5_COMMAND_STR_CASE(ARM_RQ); 528 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ); 529 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ); 530 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ); 531 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ); 532 MLX5_COMMAND_STR_CASE(CREATE_DCT); 533 MLX5_COMMAND_STR_CASE(DESTROY_DCT); 534 MLX5_COMMAND_STR_CASE(DRAIN_DCT); 535 MLX5_COMMAND_STR_CASE(QUERY_DCT); 536 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION); 537 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE); 538 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE); 539 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT); 540 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT); 541 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT); 542 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT); 543 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS); 544 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS); 545 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT); 546 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT); 547 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID); 548 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY); 549 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV); 550 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER); 551 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER); 552 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER); 553 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER); 554 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER); 555 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER); 556 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT); 557 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT); 558 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT); 559 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT); 560 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT); 561 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT); 562 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT); 563 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT); 564 MLX5_COMMAND_STR_CASE(ALLOC_PD); 565 MLX5_COMMAND_STR_CASE(DEALLOC_PD); 566 MLX5_COMMAND_STR_CASE(ALLOC_UAR); 567 MLX5_COMMAND_STR_CASE(DEALLOC_UAR); 568 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION); 569 MLX5_COMMAND_STR_CASE(ACCESS_REG); 570 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG); 571 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG); 572 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG); 573 MLX5_COMMAND_STR_CASE(MAD_IFC); 574 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX); 575 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX); 576 MLX5_COMMAND_STR_CASE(NOP); 577 MLX5_COMMAND_STR_CASE(ALLOC_XRCD); 578 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD); 579 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN); 580 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN); 581 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS); 582 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS); 583 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS); 584 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS); 585 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS); 586 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT); 587 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT); 588 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY); 589 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY); 590 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY); 591 MLX5_COMMAND_STR_CASE(SET_WOL_ROL); 592 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL); 593 MLX5_COMMAND_STR_CASE(CREATE_LAG); 594 MLX5_COMMAND_STR_CASE(MODIFY_LAG); 595 MLX5_COMMAND_STR_CASE(QUERY_LAG); 596 MLX5_COMMAND_STR_CASE(DESTROY_LAG); 597 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG); 598 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG); 599 MLX5_COMMAND_STR_CASE(CREATE_TIR); 600 MLX5_COMMAND_STR_CASE(MODIFY_TIR); 601 MLX5_COMMAND_STR_CASE(DESTROY_TIR); 602 MLX5_COMMAND_STR_CASE(QUERY_TIR); 603 MLX5_COMMAND_STR_CASE(CREATE_SQ); 604 MLX5_COMMAND_STR_CASE(MODIFY_SQ); 605 MLX5_COMMAND_STR_CASE(DESTROY_SQ); 606 MLX5_COMMAND_STR_CASE(QUERY_SQ); 607 MLX5_COMMAND_STR_CASE(CREATE_RQ); 608 MLX5_COMMAND_STR_CASE(MODIFY_RQ); 609 MLX5_COMMAND_STR_CASE(DESTROY_RQ); 610 MLX5_COMMAND_STR_CASE(QUERY_RQ); 611 MLX5_COMMAND_STR_CASE(CREATE_RMP); 612 MLX5_COMMAND_STR_CASE(MODIFY_RMP); 613 MLX5_COMMAND_STR_CASE(DESTROY_RMP); 614 MLX5_COMMAND_STR_CASE(QUERY_RMP); 615 MLX5_COMMAND_STR_CASE(CREATE_TIS); 616 MLX5_COMMAND_STR_CASE(MODIFY_TIS); 617 MLX5_COMMAND_STR_CASE(DESTROY_TIS); 618 MLX5_COMMAND_STR_CASE(QUERY_TIS); 619 MLX5_COMMAND_STR_CASE(CREATE_RQT); 620 MLX5_COMMAND_STR_CASE(MODIFY_RQT); 621 MLX5_COMMAND_STR_CASE(DESTROY_RQT); 622 MLX5_COMMAND_STR_CASE(QUERY_RQT); 623 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT); 624 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE); 625 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE); 626 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE); 627 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP); 628 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP); 629 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP); 630 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY); 631 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY); 632 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY); 633 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER); 634 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER); 635 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER); 636 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE); 637 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT); 638 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT); 639 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT); 640 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT); 641 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP); 642 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP); 643 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP); 644 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS); 645 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP); 646 MLX5_COMMAND_STR_CASE(CREATE_XRQ); 647 MLX5_COMMAND_STR_CASE(DESTROY_XRQ); 648 MLX5_COMMAND_STR_CASE(QUERY_XRQ); 649 MLX5_COMMAND_STR_CASE(ARM_XRQ); 650 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT); 651 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT); 652 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT); 653 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT); 654 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT); 655 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC); 656 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC); 657 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS); 658 MLX5_COMMAND_STR_CASE(CREATE_UCTX); 659 MLX5_COMMAND_STR_CASE(DESTROY_UCTX); 660 MLX5_COMMAND_STR_CASE(CREATE_UMEM); 661 MLX5_COMMAND_STR_CASE(DESTROY_UMEM); 662 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR); 663 MLX5_COMMAND_STR_CASE(MODIFY_XRQ); 664 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE); 665 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE); 666 MLX5_COMMAND_STR_CASE(ALLOC_SF); 667 MLX5_COMMAND_STR_CASE(DEALLOC_SF); 668 default: return "unknown command opcode"; 669 } 670 } 671 672 static const char *cmd_status_str(u8 status) 673 { 674 switch (status) { 675 case MLX5_CMD_STAT_OK: 676 return "OK"; 677 case MLX5_CMD_STAT_INT_ERR: 678 return "internal error"; 679 case MLX5_CMD_STAT_BAD_OP_ERR: 680 return "bad operation"; 681 case MLX5_CMD_STAT_BAD_PARAM_ERR: 682 return "bad parameter"; 683 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: 684 return "bad system state"; 685 case MLX5_CMD_STAT_BAD_RES_ERR: 686 return "bad resource"; 687 case MLX5_CMD_STAT_RES_BUSY: 688 return "resource busy"; 689 case MLX5_CMD_STAT_LIM_ERR: 690 return "limits exceeded"; 691 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: 692 return "bad resource state"; 693 case MLX5_CMD_STAT_IX_ERR: 694 return "bad index"; 695 case MLX5_CMD_STAT_NO_RES_ERR: 696 return "no resources"; 697 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: 698 return "bad input length"; 699 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: 700 return "bad output length"; 701 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: 702 return "bad QP state"; 703 case MLX5_CMD_STAT_BAD_PKT_ERR: 704 return "bad packet (discarded)"; 705 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: 706 return "bad size too many outstanding CQEs"; 707 default: 708 return "unknown status"; 709 } 710 } 711 712 static int cmd_status_to_err(u8 status) 713 { 714 switch (status) { 715 case MLX5_CMD_STAT_OK: return 0; 716 case MLX5_CMD_STAT_INT_ERR: return -EIO; 717 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL; 718 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL; 719 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO; 720 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL; 721 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY; 722 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM; 723 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL; 724 case MLX5_CMD_STAT_IX_ERR: return -EINVAL; 725 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN; 726 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO; 727 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO; 728 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL; 729 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL; 730 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL; 731 default: return -EIO; 732 } 733 } 734 735 struct mlx5_ifc_mbox_out_bits { 736 u8 status[0x8]; 737 u8 reserved_at_8[0x18]; 738 739 u8 syndrome[0x20]; 740 741 u8 reserved_at_40[0x40]; 742 }; 743 744 struct mlx5_ifc_mbox_in_bits { 745 u8 opcode[0x10]; 746 u8 uid[0x10]; 747 748 u8 reserved_at_20[0x10]; 749 u8 op_mod[0x10]; 750 751 u8 reserved_at_40[0x40]; 752 }; 753 754 void mlx5_cmd_mbox_status(void *out, u8 *status, u32 *syndrome) 755 { 756 *status = MLX5_GET(mbox_out, out, status); 757 *syndrome = MLX5_GET(mbox_out, out, syndrome); 758 } 759 760 static int mlx5_cmd_check(struct mlx5_core_dev *dev, void *in, void *out) 761 { 762 u32 syndrome; 763 u8 status; 764 u16 opcode; 765 u16 op_mod; 766 u16 uid; 767 768 mlx5_cmd_mbox_status(out, &status, &syndrome); 769 if (!status) 770 return 0; 771 772 opcode = MLX5_GET(mbox_in, in, opcode); 773 op_mod = MLX5_GET(mbox_in, in, op_mod); 774 uid = MLX5_GET(mbox_in, in, uid); 775 776 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY) 777 mlx5_core_err_rl(dev, 778 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x)\n", 779 mlx5_command_str(opcode), opcode, op_mod, 780 cmd_status_str(status), status, syndrome); 781 else 782 mlx5_core_dbg(dev, 783 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x)\n", 784 mlx5_command_str(opcode), 785 opcode, op_mod, 786 cmd_status_str(status), 787 status, 788 syndrome); 789 790 return cmd_status_to_err(status); 791 } 792 793 static void dump_command(struct mlx5_core_dev *dev, 794 struct mlx5_cmd_work_ent *ent, int input) 795 { 796 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out; 797 u16 op = MLX5_GET(mbox_in, ent->lay->in, opcode); 798 struct mlx5_cmd_mailbox *next = msg->next; 799 int n = mlx5_calc_cmd_blocks(msg); 800 int data_only; 801 u32 offset = 0; 802 int dump_len; 803 int i; 804 805 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx); 806 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA)); 807 808 if (data_only) 809 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA, 810 "cmd[%d]: dump command data %s(0x%x) %s\n", 811 ent->idx, mlx5_command_str(op), op, 812 input ? "INPUT" : "OUTPUT"); 813 else 814 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n", 815 ent->idx, mlx5_command_str(op), op, 816 input ? "INPUT" : "OUTPUT"); 817 818 if (data_only) { 819 if (input) { 820 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx); 821 offset += sizeof(ent->lay->in); 822 } else { 823 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx); 824 offset += sizeof(ent->lay->out); 825 } 826 } else { 827 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx); 828 offset += sizeof(*ent->lay); 829 } 830 831 for (i = 0; i < n && next; i++) { 832 if (data_only) { 833 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset); 834 dump_buf(next->buf, dump_len, 1, offset, ent->idx); 835 offset += MLX5_CMD_DATA_BLOCK_SIZE; 836 } else { 837 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx); 838 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset, 839 ent->idx); 840 offset += sizeof(struct mlx5_cmd_prot_block); 841 } 842 next = next->next; 843 } 844 845 if (data_only) 846 pr_debug("\n"); 847 848 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx); 849 } 850 851 static u16 msg_to_opcode(struct mlx5_cmd_msg *in) 852 { 853 return MLX5_GET(mbox_in, in->first.data, opcode); 854 } 855 856 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced); 857 858 static void cb_timeout_handler(struct work_struct *work) 859 { 860 struct delayed_work *dwork = container_of(work, struct delayed_work, 861 work); 862 struct mlx5_cmd_work_ent *ent = container_of(dwork, 863 struct mlx5_cmd_work_ent, 864 cb_timeout_work); 865 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, 866 cmd); 867 868 mlx5_cmd_eq_recover(dev); 869 870 /* Maybe got handled by eq recover ? */ 871 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) { 872 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx, 873 mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); 874 goto out; /* phew, already handled */ 875 } 876 877 ent->ret = -ETIMEDOUT; 878 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n", 879 ent->idx, mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); 880 mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); 881 882 out: 883 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */ 884 } 885 886 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg); 887 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, 888 struct mlx5_cmd_msg *msg); 889 890 static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode) 891 { 892 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL) 893 return true; 894 895 return cmd->allowed_opcode == opcode; 896 } 897 898 static int cmd_alloc_index_retry(struct mlx5_cmd *cmd) 899 { 900 unsigned long alloc_end = jiffies + msecs_to_jiffies(1000); 901 int idx; 902 903 retry: 904 idx = cmd_alloc_index(cmd); 905 if (idx < 0 && time_before(jiffies, alloc_end)) { 906 /* Index allocation can fail on heavy load of commands. This is a temporary 907 * situation as the current command already holds the semaphore, meaning that 908 * another command completion is being handled and it is expected to release 909 * the entry index soon. 910 */ 911 cpu_relax(); 912 goto retry; 913 } 914 return idx; 915 } 916 917 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev) 918 { 919 return pci_channel_offline(dev->pdev) || 920 dev->cmd.state != MLX5_CMDIF_STATE_UP || 921 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR; 922 } 923 924 static void cmd_work_handler(struct work_struct *work) 925 { 926 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work); 927 struct mlx5_cmd *cmd = ent->cmd; 928 struct mlx5_core_dev *dev = container_of(cmd, struct mlx5_core_dev, cmd); 929 unsigned long cb_timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC); 930 struct mlx5_cmd_layout *lay; 931 struct semaphore *sem; 932 unsigned long flags; 933 bool poll_cmd = ent->polling; 934 int alloc_ret; 935 int cmd_mode; 936 937 complete(&ent->handling); 938 sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem; 939 down(sem); 940 if (!ent->page_queue) { 941 alloc_ret = cmd_alloc_index_retry(cmd); 942 if (alloc_ret < 0) { 943 mlx5_core_err_rl(dev, "failed to allocate command entry\n"); 944 if (ent->callback) { 945 ent->callback(-EAGAIN, ent->context); 946 mlx5_free_cmd_msg(dev, ent->out); 947 free_msg(dev, ent->in); 948 cmd_ent_put(ent); 949 } else { 950 ent->ret = -EAGAIN; 951 complete(&ent->done); 952 } 953 up(sem); 954 return; 955 } 956 ent->idx = alloc_ret; 957 } else { 958 ent->idx = cmd->max_reg_cmds; 959 spin_lock_irqsave(&cmd->alloc_lock, flags); 960 clear_bit(ent->idx, &cmd->bitmask); 961 spin_unlock_irqrestore(&cmd->alloc_lock, flags); 962 } 963 964 cmd->ent_arr[ent->idx] = ent; 965 lay = get_inst(cmd, ent->idx); 966 ent->lay = lay; 967 memset(lay, 0, sizeof(*lay)); 968 memcpy(lay->in, ent->in->first.data, sizeof(lay->in)); 969 ent->op = be32_to_cpu(lay->in[0]) >> 16; 970 if (ent->in->next) 971 lay->in_ptr = cpu_to_be64(ent->in->next->dma); 972 lay->inlen = cpu_to_be32(ent->in->len); 973 if (ent->out->next) 974 lay->out_ptr = cpu_to_be64(ent->out->next->dma); 975 lay->outlen = cpu_to_be32(ent->out->len); 976 lay->type = MLX5_PCI_CMD_XPORT; 977 lay->token = ent->token; 978 lay->status_own = CMD_OWNER_HW; 979 set_signature(ent, !cmd->checksum_disabled); 980 dump_command(dev, ent, 1); 981 ent->ts1 = ktime_get_ns(); 982 cmd_mode = cmd->mode; 983 984 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, cb_timeout)) 985 cmd_ent_get(ent); 986 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state); 987 988 /* Skip sending command to fw if internal error */ 989 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) { 990 u8 status = 0; 991 u32 drv_synd; 992 993 ent->ret = mlx5_internal_err_ret_value(dev, msg_to_opcode(ent->in), &drv_synd, &status); 994 MLX5_SET(mbox_out, ent->out, status, status); 995 MLX5_SET(mbox_out, ent->out, syndrome, drv_synd); 996 997 mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); 998 return; 999 } 1000 1001 cmd_ent_get(ent); /* for the _real_ FW event on completion */ 1002 /* ring doorbell after the descriptor is valid */ 1003 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx); 1004 wmb(); 1005 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell); 1006 /* if not in polling don't use ent after this point */ 1007 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) { 1008 poll_timeout(ent); 1009 /* make sure we read the descriptor after ownership is SW */ 1010 rmb(); 1011 mlx5_cmd_comp_handler(dev, 1UL << ent->idx, (ent->ret == -ETIMEDOUT)); 1012 } 1013 } 1014 1015 static const char *deliv_status_to_str(u8 status) 1016 { 1017 switch (status) { 1018 case MLX5_CMD_DELIVERY_STAT_OK: 1019 return "no errors"; 1020 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: 1021 return "signature error"; 1022 case MLX5_CMD_DELIVERY_STAT_TOK_ERR: 1023 return "token error"; 1024 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: 1025 return "bad block number"; 1026 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: 1027 return "output pointer not aligned to block size"; 1028 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: 1029 return "input pointer not aligned to block size"; 1030 case MLX5_CMD_DELIVERY_STAT_FW_ERR: 1031 return "firmware internal error"; 1032 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: 1033 return "command input length error"; 1034 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: 1035 return "command output length error"; 1036 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: 1037 return "reserved fields not cleared"; 1038 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: 1039 return "bad command descriptor type"; 1040 default: 1041 return "unknown status code"; 1042 } 1043 } 1044 1045 enum { 1046 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000, 1047 }; 1048 1049 static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev, 1050 struct mlx5_cmd_work_ent *ent) 1051 { 1052 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC); 1053 1054 mlx5_cmd_eq_recover(dev); 1055 1056 /* Re-wait on the ent->done after executing the recovery flow. If the 1057 * recovery flow (or any other recovery flow running simultaneously) 1058 * has recovered an EQE, it should cause the entry to be completed by 1059 * the command interface. 1060 */ 1061 if (wait_for_completion_timeout(&ent->done, timeout)) { 1062 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx, 1063 mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); 1064 return; 1065 } 1066 1067 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx, 1068 mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); 1069 1070 ent->ret = -ETIMEDOUT; 1071 mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); 1072 } 1073 1074 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) 1075 { 1076 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC); 1077 struct mlx5_cmd *cmd = &dev->cmd; 1078 int err; 1079 1080 if (!wait_for_completion_timeout(&ent->handling, timeout) && 1081 cancel_work_sync(&ent->work)) { 1082 ent->ret = -ECANCELED; 1083 goto out_err; 1084 } 1085 if (cmd->mode == CMD_MODE_POLLING || ent->polling) 1086 wait_for_completion(&ent->done); 1087 else if (!wait_for_completion_timeout(&ent->done, timeout)) 1088 wait_func_handle_exec_timeout(dev, ent); 1089 1090 out_err: 1091 err = ent->ret; 1092 1093 if (err == -ETIMEDOUT) { 1094 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", 1095 mlx5_command_str(msg_to_opcode(ent->in)), 1096 msg_to_opcode(ent->in)); 1097 } else if (err == -ECANCELED) { 1098 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n", 1099 mlx5_command_str(msg_to_opcode(ent->in)), 1100 msg_to_opcode(ent->in)); 1101 } 1102 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n", 1103 err, deliv_status_to_str(ent->status), ent->status); 1104 1105 return err; 1106 } 1107 1108 /* Notes: 1109 * 1. Callback functions may not sleep 1110 * 2. page queue commands do not support asynchrous completion 1111 */ 1112 static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, 1113 struct mlx5_cmd_msg *out, void *uout, int uout_size, 1114 mlx5_cmd_cbk_t callback, 1115 void *context, int page_queue, u8 *status, 1116 u8 token, bool force_polling) 1117 { 1118 struct mlx5_cmd *cmd = &dev->cmd; 1119 struct mlx5_cmd_work_ent *ent; 1120 struct mlx5_cmd_stats *stats; 1121 int err = 0; 1122 s64 ds; 1123 u16 op; 1124 1125 if (callback && page_queue) 1126 return -EINVAL; 1127 1128 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size, 1129 callback, context, page_queue); 1130 if (IS_ERR(ent)) 1131 return PTR_ERR(ent); 1132 1133 /* put for this ent is when consumed, depending on the use case 1134 * 1) (!callback) blocking flow: by caller after wait_func completes 1135 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled 1136 */ 1137 1138 ent->token = token; 1139 ent->polling = force_polling; 1140 1141 init_completion(&ent->handling); 1142 if (!callback) 1143 init_completion(&ent->done); 1144 1145 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler); 1146 INIT_WORK(&ent->work, cmd_work_handler); 1147 if (page_queue) { 1148 cmd_work_handler(&ent->work); 1149 } else if (!queue_work(cmd->wq, &ent->work)) { 1150 mlx5_core_warn(dev, "failed to queue work\n"); 1151 err = -ENOMEM; 1152 goto out_free; 1153 } 1154 1155 if (callback) 1156 goto out; /* mlx5_cmd_comp_handler() will put(ent) */ 1157 1158 err = wait_func(dev, ent); 1159 if (err == -ETIMEDOUT || err == -ECANCELED) 1160 goto out_free; 1161 1162 ds = ent->ts2 - ent->ts1; 1163 op = MLX5_GET(mbox_in, in->first.data, opcode); 1164 if (op < MLX5_CMD_OP_MAX) { 1165 stats = &cmd->stats[op]; 1166 spin_lock_irq(&stats->lock); 1167 stats->sum += ds; 1168 ++stats->n; 1169 spin_unlock_irq(&stats->lock); 1170 } 1171 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME, 1172 "fw exec time for %s is %lld nsec\n", 1173 mlx5_command_str(op), ds); 1174 *status = ent->status; 1175 1176 out_free: 1177 cmd_ent_put(ent); 1178 out: 1179 return err; 1180 } 1181 1182 static ssize_t dbg_write(struct file *filp, const char __user *buf, 1183 size_t count, loff_t *pos) 1184 { 1185 struct mlx5_core_dev *dev = filp->private_data; 1186 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1187 char lbuf[3]; 1188 int err; 1189 1190 if (!dbg->in_msg || !dbg->out_msg) 1191 return -ENOMEM; 1192 1193 if (count < sizeof(lbuf) - 1) 1194 return -EINVAL; 1195 1196 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1)) 1197 return -EFAULT; 1198 1199 lbuf[sizeof(lbuf) - 1] = 0; 1200 1201 if (strcmp(lbuf, "go")) 1202 return -EINVAL; 1203 1204 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen); 1205 1206 return err ? err : count; 1207 } 1208 1209 static const struct file_operations fops = { 1210 .owner = THIS_MODULE, 1211 .open = simple_open, 1212 .write = dbg_write, 1213 }; 1214 1215 static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size, 1216 u8 token) 1217 { 1218 struct mlx5_cmd_prot_block *block; 1219 struct mlx5_cmd_mailbox *next; 1220 int copy; 1221 1222 if (!to || !from) 1223 return -ENOMEM; 1224 1225 copy = min_t(int, size, sizeof(to->first.data)); 1226 memcpy(to->first.data, from, copy); 1227 size -= copy; 1228 from += copy; 1229 1230 next = to->next; 1231 while (size) { 1232 if (!next) { 1233 /* this is a BUG */ 1234 return -ENOMEM; 1235 } 1236 1237 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE); 1238 block = next->buf; 1239 memcpy(block->data, from, copy); 1240 from += copy; 1241 size -= copy; 1242 block->token = token; 1243 next = next->next; 1244 } 1245 1246 return 0; 1247 } 1248 1249 static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size) 1250 { 1251 struct mlx5_cmd_prot_block *block; 1252 struct mlx5_cmd_mailbox *next; 1253 int copy; 1254 1255 if (!to || !from) 1256 return -ENOMEM; 1257 1258 copy = min_t(int, size, sizeof(from->first.data)); 1259 memcpy(to, from->first.data, copy); 1260 size -= copy; 1261 to += copy; 1262 1263 next = from->next; 1264 while (size) { 1265 if (!next) { 1266 /* this is a BUG */ 1267 return -ENOMEM; 1268 } 1269 1270 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE); 1271 block = next->buf; 1272 1273 memcpy(to, block->data, copy); 1274 to += copy; 1275 size -= copy; 1276 next = next->next; 1277 } 1278 1279 return 0; 1280 } 1281 1282 static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev, 1283 gfp_t flags) 1284 { 1285 struct mlx5_cmd_mailbox *mailbox; 1286 1287 mailbox = kmalloc(sizeof(*mailbox), flags); 1288 if (!mailbox) 1289 return ERR_PTR(-ENOMEM); 1290 1291 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags, 1292 &mailbox->dma); 1293 if (!mailbox->buf) { 1294 mlx5_core_dbg(dev, "failed allocation\n"); 1295 kfree(mailbox); 1296 return ERR_PTR(-ENOMEM); 1297 } 1298 mailbox->next = NULL; 1299 1300 return mailbox; 1301 } 1302 1303 static void free_cmd_box(struct mlx5_core_dev *dev, 1304 struct mlx5_cmd_mailbox *mailbox) 1305 { 1306 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma); 1307 kfree(mailbox); 1308 } 1309 1310 static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev, 1311 gfp_t flags, int size, 1312 u8 token) 1313 { 1314 struct mlx5_cmd_mailbox *tmp, *head = NULL; 1315 struct mlx5_cmd_prot_block *block; 1316 struct mlx5_cmd_msg *msg; 1317 int err; 1318 int n; 1319 int i; 1320 1321 msg = kzalloc(sizeof(*msg), flags); 1322 if (!msg) 1323 return ERR_PTR(-ENOMEM); 1324 1325 msg->len = size; 1326 n = mlx5_calc_cmd_blocks(msg); 1327 1328 for (i = 0; i < n; i++) { 1329 tmp = alloc_cmd_box(dev, flags); 1330 if (IS_ERR(tmp)) { 1331 mlx5_core_warn(dev, "failed allocating block\n"); 1332 err = PTR_ERR(tmp); 1333 goto err_alloc; 1334 } 1335 1336 block = tmp->buf; 1337 tmp->next = head; 1338 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0); 1339 block->block_num = cpu_to_be32(n - i - 1); 1340 block->token = token; 1341 head = tmp; 1342 } 1343 msg->next = head; 1344 return msg; 1345 1346 err_alloc: 1347 while (head) { 1348 tmp = head->next; 1349 free_cmd_box(dev, head); 1350 head = tmp; 1351 } 1352 kfree(msg); 1353 1354 return ERR_PTR(err); 1355 } 1356 1357 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, 1358 struct mlx5_cmd_msg *msg) 1359 { 1360 struct mlx5_cmd_mailbox *head = msg->next; 1361 struct mlx5_cmd_mailbox *next; 1362 1363 while (head) { 1364 next = head->next; 1365 free_cmd_box(dev, head); 1366 head = next; 1367 } 1368 kfree(msg); 1369 } 1370 1371 static ssize_t data_write(struct file *filp, const char __user *buf, 1372 size_t count, loff_t *pos) 1373 { 1374 struct mlx5_core_dev *dev = filp->private_data; 1375 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1376 void *ptr; 1377 1378 if (*pos != 0) 1379 return -EINVAL; 1380 1381 kfree(dbg->in_msg); 1382 dbg->in_msg = NULL; 1383 dbg->inlen = 0; 1384 ptr = memdup_user(buf, count); 1385 if (IS_ERR(ptr)) 1386 return PTR_ERR(ptr); 1387 dbg->in_msg = ptr; 1388 dbg->inlen = count; 1389 1390 *pos = count; 1391 1392 return count; 1393 } 1394 1395 static ssize_t data_read(struct file *filp, char __user *buf, size_t count, 1396 loff_t *pos) 1397 { 1398 struct mlx5_core_dev *dev = filp->private_data; 1399 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1400 1401 if (!dbg->out_msg) 1402 return -ENOMEM; 1403 1404 return simple_read_from_buffer(buf, count, pos, dbg->out_msg, 1405 dbg->outlen); 1406 } 1407 1408 static const struct file_operations dfops = { 1409 .owner = THIS_MODULE, 1410 .open = simple_open, 1411 .write = data_write, 1412 .read = data_read, 1413 }; 1414 1415 static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count, 1416 loff_t *pos) 1417 { 1418 struct mlx5_core_dev *dev = filp->private_data; 1419 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1420 char outlen[8]; 1421 int err; 1422 1423 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen); 1424 if (err < 0) 1425 return err; 1426 1427 return simple_read_from_buffer(buf, count, pos, outlen, err); 1428 } 1429 1430 static ssize_t outlen_write(struct file *filp, const char __user *buf, 1431 size_t count, loff_t *pos) 1432 { 1433 struct mlx5_core_dev *dev = filp->private_data; 1434 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1435 char outlen_str[8] = {0}; 1436 int outlen; 1437 void *ptr; 1438 int err; 1439 1440 if (*pos != 0 || count > 6) 1441 return -EINVAL; 1442 1443 kfree(dbg->out_msg); 1444 dbg->out_msg = NULL; 1445 dbg->outlen = 0; 1446 1447 if (copy_from_user(outlen_str, buf, count)) 1448 return -EFAULT; 1449 1450 err = sscanf(outlen_str, "%d", &outlen); 1451 if (err < 0) 1452 return err; 1453 1454 ptr = kzalloc(outlen, GFP_KERNEL); 1455 if (!ptr) 1456 return -ENOMEM; 1457 1458 dbg->out_msg = ptr; 1459 dbg->outlen = outlen; 1460 1461 *pos = count; 1462 1463 return count; 1464 } 1465 1466 static const struct file_operations olfops = { 1467 .owner = THIS_MODULE, 1468 .open = simple_open, 1469 .write = outlen_write, 1470 .read = outlen_read, 1471 }; 1472 1473 static void set_wqname(struct mlx5_core_dev *dev) 1474 { 1475 struct mlx5_cmd *cmd = &dev->cmd; 1476 1477 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s", 1478 dev_name(dev->device)); 1479 } 1480 1481 static void clean_debug_files(struct mlx5_core_dev *dev) 1482 { 1483 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1484 1485 if (!mlx5_debugfs_root) 1486 return; 1487 1488 mlx5_cmdif_debugfs_cleanup(dev); 1489 debugfs_remove_recursive(dbg->dbg_root); 1490 } 1491 1492 static void create_debugfs_files(struct mlx5_core_dev *dev) 1493 { 1494 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg; 1495 1496 dbg->dbg_root = debugfs_create_dir("cmd", dev->priv.dbg_root); 1497 1498 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops); 1499 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops); 1500 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops); 1501 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status); 1502 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops); 1503 1504 mlx5_cmdif_debugfs_init(dev); 1505 } 1506 1507 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode) 1508 { 1509 struct mlx5_cmd *cmd = &dev->cmd; 1510 int i; 1511 1512 for (i = 0; i < cmd->max_reg_cmds; i++) 1513 down(&cmd->sem); 1514 down(&cmd->pages_sem); 1515 1516 cmd->allowed_opcode = opcode; 1517 1518 up(&cmd->pages_sem); 1519 for (i = 0; i < cmd->max_reg_cmds; i++) 1520 up(&cmd->sem); 1521 } 1522 1523 static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) 1524 { 1525 struct mlx5_cmd *cmd = &dev->cmd; 1526 int i; 1527 1528 for (i = 0; i < cmd->max_reg_cmds; i++) 1529 down(&cmd->sem); 1530 down(&cmd->pages_sem); 1531 1532 cmd->mode = mode; 1533 1534 up(&cmd->pages_sem); 1535 for (i = 0; i < cmd->max_reg_cmds; i++) 1536 up(&cmd->sem); 1537 } 1538 1539 static int cmd_comp_notifier(struct notifier_block *nb, 1540 unsigned long type, void *data) 1541 { 1542 struct mlx5_core_dev *dev; 1543 struct mlx5_cmd *cmd; 1544 struct mlx5_eqe *eqe; 1545 1546 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb); 1547 dev = container_of(cmd, struct mlx5_core_dev, cmd); 1548 eqe = data; 1549 1550 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false); 1551 1552 return NOTIFY_OK; 1553 } 1554 void mlx5_cmd_use_events(struct mlx5_core_dev *dev) 1555 { 1556 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD); 1557 mlx5_eq_notifier_register(dev, &dev->cmd.nb); 1558 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS); 1559 } 1560 1561 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev) 1562 { 1563 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING); 1564 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb); 1565 } 1566 1567 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg) 1568 { 1569 unsigned long flags; 1570 1571 if (msg->parent) { 1572 spin_lock_irqsave(&msg->parent->lock, flags); 1573 list_add_tail(&msg->list, &msg->parent->head); 1574 spin_unlock_irqrestore(&msg->parent->lock, flags); 1575 } else { 1576 mlx5_free_cmd_msg(dev, msg); 1577 } 1578 } 1579 1580 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced) 1581 { 1582 struct mlx5_cmd *cmd = &dev->cmd; 1583 struct mlx5_cmd_work_ent *ent; 1584 mlx5_cmd_cbk_t callback; 1585 void *context; 1586 int err; 1587 int i; 1588 s64 ds; 1589 struct mlx5_cmd_stats *stats; 1590 unsigned long flags; 1591 unsigned long vector; 1592 1593 /* there can be at most 32 command queues */ 1594 vector = vec & 0xffffffff; 1595 for (i = 0; i < (1 << cmd->log_sz); i++) { 1596 if (test_bit(i, &vector)) { 1597 struct semaphore *sem; 1598 1599 ent = cmd->ent_arr[i]; 1600 1601 /* if we already completed the command, ignore it */ 1602 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, 1603 &ent->state)) { 1604 /* only real completion can free the cmd slot */ 1605 if (!forced) { 1606 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n", 1607 ent->idx); 1608 cmd_ent_put(ent); 1609 } 1610 continue; 1611 } 1612 1613 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work)) 1614 cmd_ent_put(ent); /* timeout work was canceled */ 1615 1616 if (!forced || /* Real FW completion */ 1617 pci_channel_offline(dev->pdev) || /* FW is inaccessible */ 1618 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 1619 cmd_ent_put(ent); 1620 1621 if (ent->page_queue) 1622 sem = &cmd->pages_sem; 1623 else 1624 sem = &cmd->sem; 1625 ent->ts2 = ktime_get_ns(); 1626 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out)); 1627 dump_command(dev, ent, 0); 1628 if (!ent->ret) { 1629 if (!cmd->checksum_disabled) 1630 ent->ret = verify_signature(ent); 1631 else 1632 ent->ret = 0; 1633 if (vec & MLX5_TRIGGERED_CMD_COMP) 1634 ent->status = MLX5_DRIVER_STATUS_ABORTED; 1635 else 1636 ent->status = ent->lay->status_own >> 1; 1637 1638 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n", 1639 ent->ret, deliv_status_to_str(ent->status), ent->status); 1640 } 1641 1642 if (ent->callback) { 1643 ds = ent->ts2 - ent->ts1; 1644 if (ent->op < MLX5_CMD_OP_MAX) { 1645 stats = &cmd->stats[ent->op]; 1646 spin_lock_irqsave(&stats->lock, flags); 1647 stats->sum += ds; 1648 ++stats->n; 1649 spin_unlock_irqrestore(&stats->lock, flags); 1650 } 1651 1652 callback = ent->callback; 1653 context = ent->context; 1654 err = ent->ret; 1655 if (!err) { 1656 err = mlx5_copy_from_msg(ent->uout, 1657 ent->out, 1658 ent->uout_size); 1659 1660 err = err ? err : mlx5_cmd_check(dev, 1661 ent->in->first.data, 1662 ent->uout); 1663 } 1664 1665 mlx5_free_cmd_msg(dev, ent->out); 1666 free_msg(dev, ent->in); 1667 1668 err = err ? err : ent->status; 1669 /* final consumer is done, release ent */ 1670 cmd_ent_put(ent); 1671 callback(err, context); 1672 } else { 1673 /* release wait_func() so mlx5_cmd_invoke() 1674 * can make the final ent_put() 1675 */ 1676 complete(&ent->done); 1677 } 1678 up(sem); 1679 } 1680 } 1681 } 1682 1683 void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev) 1684 { 1685 struct mlx5_cmd *cmd = &dev->cmd; 1686 unsigned long bitmask; 1687 unsigned long flags; 1688 u64 vector; 1689 int i; 1690 1691 /* wait for pending handlers to complete */ 1692 mlx5_eq_synchronize_cmd_irq(dev); 1693 spin_lock_irqsave(&dev->cmd.alloc_lock, flags); 1694 vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1); 1695 if (!vector) 1696 goto no_trig; 1697 1698 bitmask = vector; 1699 /* we must increment the allocated entries refcount before triggering the completions 1700 * to guarantee pending commands will not get freed in the meanwhile. 1701 * For that reason, it also has to be done inside the alloc_lock. 1702 */ 1703 for_each_set_bit(i, &bitmask, (1 << cmd->log_sz)) 1704 cmd_ent_get(cmd->ent_arr[i]); 1705 vector |= MLX5_TRIGGERED_CMD_COMP; 1706 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 1707 1708 mlx5_core_dbg(dev, "vector 0x%llx\n", vector); 1709 mlx5_cmd_comp_handler(dev, vector, true); 1710 for_each_set_bit(i, &bitmask, (1 << cmd->log_sz)) 1711 cmd_ent_put(cmd->ent_arr[i]); 1712 return; 1713 1714 no_trig: 1715 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 1716 } 1717 1718 void mlx5_cmd_flush(struct mlx5_core_dev *dev) 1719 { 1720 struct mlx5_cmd *cmd = &dev->cmd; 1721 int i; 1722 1723 for (i = 0; i < cmd->max_reg_cmds; i++) 1724 while (down_trylock(&cmd->sem)) 1725 mlx5_cmd_trigger_completions(dev); 1726 1727 while (down_trylock(&cmd->pages_sem)) 1728 mlx5_cmd_trigger_completions(dev); 1729 1730 /* Unlock cmdif */ 1731 up(&cmd->pages_sem); 1732 for (i = 0; i < cmd->max_reg_cmds; i++) 1733 up(&cmd->sem); 1734 } 1735 1736 static int status_to_err(u8 status) 1737 { 1738 switch (status) { 1739 case MLX5_CMD_DELIVERY_STAT_OK: 1740 case MLX5_DRIVER_STATUS_ABORTED: 1741 return 0; 1742 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR: 1743 case MLX5_CMD_DELIVERY_STAT_TOK_ERR: 1744 return -EBADR; 1745 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR: 1746 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR: 1747 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR: 1748 return -EFAULT; /* Bad address */ 1749 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR: 1750 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR: 1751 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR: 1752 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR: 1753 return -ENOMSG; 1754 case MLX5_CMD_DELIVERY_STAT_FW_ERR: 1755 return -EIO; 1756 default: 1757 return -EINVAL; 1758 } 1759 } 1760 1761 static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size, 1762 gfp_t gfp) 1763 { 1764 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM); 1765 struct cmd_msg_cache *ch = NULL; 1766 struct mlx5_cmd *cmd = &dev->cmd; 1767 int i; 1768 1769 if (in_size <= 16) 1770 goto cache_miss; 1771 1772 for (i = 0; i < MLX5_NUM_COMMAND_CACHES; i++) { 1773 ch = &cmd->cache[i]; 1774 if (in_size > ch->max_inbox_size) 1775 continue; 1776 spin_lock_irq(&ch->lock); 1777 if (list_empty(&ch->head)) { 1778 spin_unlock_irq(&ch->lock); 1779 continue; 1780 } 1781 msg = list_entry(ch->head.next, typeof(*msg), list); 1782 /* For cached lists, we must explicitly state what is 1783 * the real size 1784 */ 1785 msg->len = in_size; 1786 list_del(&msg->list); 1787 spin_unlock_irq(&ch->lock); 1788 break; 1789 } 1790 1791 if (!IS_ERR(msg)) 1792 return msg; 1793 1794 cache_miss: 1795 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0); 1796 return msg; 1797 } 1798 1799 static int is_manage_pages(void *in) 1800 { 1801 return MLX5_GET(mbox_in, in, opcode) == MLX5_CMD_OP_MANAGE_PAGES; 1802 } 1803 1804 static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 1805 int out_size, mlx5_cmd_cbk_t callback, void *context, 1806 bool force_polling) 1807 { 1808 struct mlx5_cmd_msg *inb; 1809 struct mlx5_cmd_msg *outb; 1810 int pages_queue; 1811 gfp_t gfp; 1812 int err; 1813 u8 status = 0; 1814 u32 drv_synd; 1815 u16 opcode; 1816 u8 token; 1817 1818 opcode = MLX5_GET(mbox_in, in, opcode); 1819 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode)) { 1820 err = mlx5_internal_err_ret_value(dev, opcode, &drv_synd, &status); 1821 MLX5_SET(mbox_out, out, status, status); 1822 MLX5_SET(mbox_out, out, syndrome, drv_synd); 1823 return err; 1824 } 1825 1826 pages_queue = is_manage_pages(in); 1827 gfp = callback ? GFP_ATOMIC : GFP_KERNEL; 1828 1829 inb = alloc_msg(dev, in_size, gfp); 1830 if (IS_ERR(inb)) { 1831 err = PTR_ERR(inb); 1832 return err; 1833 } 1834 1835 token = alloc_token(&dev->cmd); 1836 1837 err = mlx5_copy_to_msg(inb, in, in_size, token); 1838 if (err) { 1839 mlx5_core_warn(dev, "err %d\n", err); 1840 goto out_in; 1841 } 1842 1843 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token); 1844 if (IS_ERR(outb)) { 1845 err = PTR_ERR(outb); 1846 goto out_in; 1847 } 1848 1849 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context, 1850 pages_queue, &status, token, force_polling); 1851 if (err) 1852 goto out_out; 1853 1854 mlx5_core_dbg(dev, "err %d, status %d\n", err, status); 1855 if (status) { 1856 err = status_to_err(status); 1857 goto out_out; 1858 } 1859 1860 if (!callback) 1861 err = mlx5_copy_from_msg(out, outb, out_size); 1862 1863 out_out: 1864 if (!callback) 1865 mlx5_free_cmd_msg(dev, outb); 1866 1867 out_in: 1868 if (!callback) 1869 free_msg(dev, inb); 1870 return err; 1871 } 1872 1873 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 1874 int out_size) 1875 { 1876 int err; 1877 1878 err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false); 1879 return err ? : mlx5_cmd_check(dev, in, out); 1880 } 1881 EXPORT_SYMBOL(mlx5_cmd_exec); 1882 1883 void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev, 1884 struct mlx5_async_ctx *ctx) 1885 { 1886 ctx->dev = dev; 1887 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */ 1888 atomic_set(&ctx->num_inflight, 1); 1889 init_waitqueue_head(&ctx->wait); 1890 } 1891 EXPORT_SYMBOL(mlx5_cmd_init_async_ctx); 1892 1893 /** 1894 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx 1895 * @ctx: The ctx to clean 1896 * 1897 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The 1898 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after 1899 * the call mlx5_cleanup_async_ctx(). 1900 */ 1901 void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx) 1902 { 1903 atomic_dec(&ctx->num_inflight); 1904 wait_event(ctx->wait, atomic_read(&ctx->num_inflight) == 0); 1905 } 1906 EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx); 1907 1908 static void mlx5_cmd_exec_cb_handler(int status, void *_work) 1909 { 1910 struct mlx5_async_work *work = _work; 1911 struct mlx5_async_ctx *ctx = work->ctx; 1912 1913 work->user_callback(status, work); 1914 if (atomic_dec_and_test(&ctx->num_inflight)) 1915 wake_up(&ctx->wait); 1916 } 1917 1918 int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, 1919 void *out, int out_size, mlx5_async_cbk_t callback, 1920 struct mlx5_async_work *work) 1921 { 1922 int ret; 1923 1924 work->ctx = ctx; 1925 work->user_callback = callback; 1926 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight))) 1927 return -EIO; 1928 ret = cmd_exec(ctx->dev, in, in_size, out, out_size, 1929 mlx5_cmd_exec_cb_handler, work, false); 1930 if (ret && atomic_dec_and_test(&ctx->num_inflight)) 1931 wake_up(&ctx->wait); 1932 1933 return ret; 1934 } 1935 EXPORT_SYMBOL(mlx5_cmd_exec_cb); 1936 1937 int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, 1938 void *out, int out_size) 1939 { 1940 int err; 1941 1942 err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true); 1943 1944 return err ? : mlx5_cmd_check(dev, in, out); 1945 } 1946 EXPORT_SYMBOL(mlx5_cmd_exec_polling); 1947 1948 static void destroy_msg_cache(struct mlx5_core_dev *dev) 1949 { 1950 struct cmd_msg_cache *ch; 1951 struct mlx5_cmd_msg *msg; 1952 struct mlx5_cmd_msg *n; 1953 int i; 1954 1955 for (i = 0; i < MLX5_NUM_COMMAND_CACHES; i++) { 1956 ch = &dev->cmd.cache[i]; 1957 list_for_each_entry_safe(msg, n, &ch->head, list) { 1958 list_del(&msg->list); 1959 mlx5_free_cmd_msg(dev, msg); 1960 } 1961 } 1962 } 1963 1964 static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = { 1965 512, 32, 16, 8, 2 1966 }; 1967 1968 static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = { 1969 16 + MLX5_CMD_DATA_BLOCK_SIZE, 1970 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2, 1971 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16, 1972 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256, 1973 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512, 1974 }; 1975 1976 static void create_msg_cache(struct mlx5_core_dev *dev) 1977 { 1978 struct mlx5_cmd *cmd = &dev->cmd; 1979 struct cmd_msg_cache *ch; 1980 struct mlx5_cmd_msg *msg; 1981 int i; 1982 int k; 1983 1984 /* Initialize and fill the caches with initial entries */ 1985 for (k = 0; k < MLX5_NUM_COMMAND_CACHES; k++) { 1986 ch = &cmd->cache[k]; 1987 spin_lock_init(&ch->lock); 1988 INIT_LIST_HEAD(&ch->head); 1989 ch->num_ent = cmd_cache_num_ent[k]; 1990 ch->max_inbox_size = cmd_cache_ent_size[k]; 1991 for (i = 0; i < ch->num_ent; i++) { 1992 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN, 1993 ch->max_inbox_size, 0); 1994 if (IS_ERR(msg)) 1995 break; 1996 msg->parent = ch; 1997 list_add_tail(&msg->list, &ch->head); 1998 } 1999 } 2000 } 2001 2002 static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) 2003 { 2004 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, 2005 &cmd->alloc_dma, GFP_KERNEL); 2006 if (!cmd->cmd_alloc_buf) 2007 return -ENOMEM; 2008 2009 /* make sure it is aligned to 4K */ 2010 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) { 2011 cmd->cmd_buf = cmd->cmd_alloc_buf; 2012 cmd->dma = cmd->alloc_dma; 2013 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE; 2014 return 0; 2015 } 2016 2017 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf, 2018 cmd->alloc_dma); 2019 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), 2020 2 * MLX5_ADAPTER_PAGE_SIZE - 1, 2021 &cmd->alloc_dma, GFP_KERNEL); 2022 if (!cmd->cmd_alloc_buf) 2023 return -ENOMEM; 2024 2025 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE); 2026 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE); 2027 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1; 2028 return 0; 2029 } 2030 2031 static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd) 2032 { 2033 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf, 2034 cmd->alloc_dma); 2035 } 2036 2037 static u16 cmdif_rev(struct mlx5_core_dev *dev) 2038 { 2039 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; 2040 } 2041 2042 int mlx5_cmd_init(struct mlx5_core_dev *dev) 2043 { 2044 int size = sizeof(struct mlx5_cmd_prot_block); 2045 int align = roundup_pow_of_two(size); 2046 struct mlx5_cmd *cmd = &dev->cmd; 2047 u32 cmd_h, cmd_l; 2048 u16 cmd_if_rev; 2049 int err; 2050 int i; 2051 2052 memset(cmd, 0, sizeof(*cmd)); 2053 cmd_if_rev = cmdif_rev(dev); 2054 if (cmd_if_rev != CMD_IF_REV) { 2055 mlx5_core_err(dev, 2056 "Driver cmdif rev(%d) differs from firmware's(%d)\n", 2057 CMD_IF_REV, cmd_if_rev); 2058 return -EINVAL; 2059 } 2060 2061 cmd->stats = kvzalloc(MLX5_CMD_OP_MAX * sizeof(*cmd->stats), GFP_KERNEL); 2062 if (!cmd->stats) 2063 return -ENOMEM; 2064 2065 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0); 2066 if (!cmd->pool) { 2067 err = -ENOMEM; 2068 goto dma_pool_err; 2069 } 2070 2071 err = alloc_cmd_page(dev, cmd); 2072 if (err) 2073 goto err_free_pool; 2074 2075 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff; 2076 cmd->log_sz = cmd_l >> 4 & 0xf; 2077 cmd->log_stride = cmd_l & 0xf; 2078 if (1 << cmd->log_sz > MLX5_MAX_COMMANDS) { 2079 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n", 2080 1 << cmd->log_sz); 2081 err = -EINVAL; 2082 goto err_free_page; 2083 } 2084 2085 if (cmd->log_sz + cmd->log_stride > MLX5_ADAPTER_PAGE_SHIFT) { 2086 mlx5_core_err(dev, "command queue size overflow\n"); 2087 err = -EINVAL; 2088 goto err_free_page; 2089 } 2090 2091 cmd->state = MLX5_CMDIF_STATE_DOWN; 2092 cmd->checksum_disabled = 1; 2093 cmd->max_reg_cmds = (1 << cmd->log_sz) - 1; 2094 cmd->bitmask = (1UL << cmd->max_reg_cmds) - 1; 2095 2096 cmd->cmdif_rev = ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16; 2097 if (cmd->cmdif_rev > CMD_IF_REV) { 2098 mlx5_core_err(dev, "driver does not support command interface version. driver %d, firmware %d\n", 2099 CMD_IF_REV, cmd->cmdif_rev); 2100 err = -EOPNOTSUPP; 2101 goto err_free_page; 2102 } 2103 2104 spin_lock_init(&cmd->alloc_lock); 2105 spin_lock_init(&cmd->token_lock); 2106 for (i = 0; i < MLX5_CMD_OP_MAX; i++) 2107 spin_lock_init(&cmd->stats[i].lock); 2108 2109 sema_init(&cmd->sem, cmd->max_reg_cmds); 2110 sema_init(&cmd->pages_sem, 1); 2111 2112 cmd_h = (u32)((u64)(cmd->dma) >> 32); 2113 cmd_l = (u32)(cmd->dma); 2114 if (cmd_l & 0xfff) { 2115 mlx5_core_err(dev, "invalid command queue address\n"); 2116 err = -ENOMEM; 2117 goto err_free_page; 2118 } 2119 2120 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h); 2121 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz); 2122 2123 /* Make sure firmware sees the complete address before we proceed */ 2124 wmb(); 2125 2126 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); 2127 2128 cmd->mode = CMD_MODE_POLLING; 2129 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL; 2130 2131 create_msg_cache(dev); 2132 2133 set_wqname(dev); 2134 cmd->wq = create_singlethread_workqueue(cmd->wq_name); 2135 if (!cmd->wq) { 2136 mlx5_core_err(dev, "failed to create command workqueue\n"); 2137 err = -ENOMEM; 2138 goto err_cache; 2139 } 2140 2141 create_debugfs_files(dev); 2142 2143 return 0; 2144 2145 err_cache: 2146 destroy_msg_cache(dev); 2147 2148 err_free_page: 2149 free_cmd_page(dev, cmd); 2150 2151 err_free_pool: 2152 dma_pool_destroy(cmd->pool); 2153 dma_pool_err: 2154 kvfree(cmd->stats); 2155 return err; 2156 } 2157 2158 void mlx5_cmd_cleanup(struct mlx5_core_dev *dev) 2159 { 2160 struct mlx5_cmd *cmd = &dev->cmd; 2161 2162 clean_debug_files(dev); 2163 destroy_workqueue(cmd->wq); 2164 destroy_msg_cache(dev); 2165 free_cmd_page(dev, cmd); 2166 dma_pool_destroy(cmd->pool); 2167 kvfree(cmd->stats); 2168 } 2169 2170 void mlx5_cmd_set_state(struct mlx5_core_dev *dev, 2171 enum mlx5_cmdif_state cmdif_state) 2172 { 2173 dev->cmd.state = cmdif_state; 2174 } 2175