1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/mlx5/driver.h> 34 #include "mlx5_core.h" 35 #include <linux/mlx5/transobj.h> 36 37 int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn) 38 { 39 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {}; 40 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {}; 41 int err; 42 43 MLX5_SET(alloc_transport_domain_in, in, opcode, 44 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN); 45 46 err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out); 47 if (!err) 48 *tdn = MLX5_GET(alloc_transport_domain_out, out, 49 transport_domain); 50 51 return err; 52 } 53 EXPORT_SYMBOL(mlx5_core_alloc_transport_domain); 54 55 void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn) 56 { 57 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {}; 58 59 MLX5_SET(dealloc_transport_domain_in, in, opcode, 60 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN); 61 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn); 62 mlx5_cmd_exec_in(dev, dealloc_transport_domain, in); 63 } 64 EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain); 65 66 int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn) 67 { 68 u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {}; 69 int err; 70 71 MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ); 72 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 73 if (!err) 74 *rqn = MLX5_GET(create_rq_out, out, rqn); 75 76 return err; 77 } 78 EXPORT_SYMBOL(mlx5_core_create_rq); 79 80 int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in) 81 { 82 MLX5_SET(modify_rq_in, in, rqn, rqn); 83 MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ); 84 85 return mlx5_cmd_exec_in(dev, modify_rq, in); 86 } 87 EXPORT_SYMBOL(mlx5_core_modify_rq); 88 89 void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn) 90 { 91 u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {}; 92 93 MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ); 94 MLX5_SET(destroy_rq_in, in, rqn, rqn); 95 mlx5_cmd_exec_in(dev, destroy_rq, in); 96 } 97 EXPORT_SYMBOL(mlx5_core_destroy_rq); 98 99 int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out) 100 { 101 u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {}; 102 103 MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ); 104 MLX5_SET(query_rq_in, in, rqn, rqn); 105 106 return mlx5_cmd_exec_inout(dev, query_rq, in, out); 107 } 108 EXPORT_SYMBOL(mlx5_core_query_rq); 109 110 int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn) 111 { 112 u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {}; 113 int err; 114 115 MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ); 116 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 117 if (!err) 118 *sqn = MLX5_GET(create_sq_out, out, sqn); 119 120 return err; 121 } 122 123 int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in) 124 { 125 MLX5_SET(modify_sq_in, in, sqn, sqn); 126 MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ); 127 return mlx5_cmd_exec_in(dev, modify_sq, in); 128 } 129 EXPORT_SYMBOL(mlx5_core_modify_sq); 130 131 void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn) 132 { 133 u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {}; 134 135 MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ); 136 MLX5_SET(destroy_sq_in, in, sqn, sqn); 137 mlx5_cmd_exec_in(dev, destroy_sq, in); 138 } 139 140 int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out) 141 { 142 u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {}; 143 144 MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ); 145 MLX5_SET(query_sq_in, in, sqn, sqn); 146 return mlx5_cmd_exec_inout(dev, query_sq, in, out); 147 } 148 EXPORT_SYMBOL(mlx5_core_query_sq); 149 150 int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state) 151 { 152 void *out; 153 void *sqc; 154 int inlen; 155 int err; 156 157 inlen = MLX5_ST_SZ_BYTES(query_sq_out); 158 out = kvzalloc(inlen, GFP_KERNEL); 159 if (!out) 160 return -ENOMEM; 161 162 err = mlx5_core_query_sq(dev, sqn, out); 163 if (err) 164 goto out; 165 166 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context); 167 *state = MLX5_GET(sqc, sqc, state); 168 169 out: 170 kvfree(out); 171 return err; 172 } 173 EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state); 174 175 int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, u32 *tirn) 176 { 177 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {}; 178 int err; 179 180 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR); 181 err = mlx5_cmd_exec_inout(dev, create_tir, in, out); 182 if (!err) 183 *tirn = MLX5_GET(create_tir_out, out, tirn); 184 185 return err; 186 } 187 EXPORT_SYMBOL(mlx5_core_create_tir); 188 189 int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in) 190 { 191 MLX5_SET(modify_tir_in, in, tirn, tirn); 192 MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR); 193 return mlx5_cmd_exec_in(dev, modify_tir, in); 194 } 195 196 void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn) 197 { 198 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {}; 199 200 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR); 201 MLX5_SET(destroy_tir_in, in, tirn, tirn); 202 mlx5_cmd_exec_in(dev, destroy_tir, in); 203 } 204 EXPORT_SYMBOL(mlx5_core_destroy_tir); 205 206 int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, u32 *tisn) 207 { 208 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {}; 209 int err; 210 211 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS); 212 err = mlx5_cmd_exec_inout(dev, create_tis, in, out); 213 if (!err) 214 *tisn = MLX5_GET(create_tis_out, out, tisn); 215 216 return err; 217 } 218 EXPORT_SYMBOL(mlx5_core_create_tis); 219 220 int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in) 221 { 222 MLX5_SET(modify_tis_in, in, tisn, tisn); 223 MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS); 224 225 return mlx5_cmd_exec_in(dev, modify_tis, in); 226 } 227 EXPORT_SYMBOL(mlx5_core_modify_tis); 228 229 void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn) 230 { 231 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {}; 232 233 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS); 234 MLX5_SET(destroy_tis_in, in, tisn, tisn); 235 mlx5_cmd_exec_in(dev, destroy_tis, in); 236 } 237 EXPORT_SYMBOL(mlx5_core_destroy_tis); 238 239 int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen, 240 u32 *rqtn) 241 { 242 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {}; 243 int err; 244 245 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT); 246 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 247 if (!err) 248 *rqtn = MLX5_GET(create_rqt_out, out, rqtn); 249 250 return err; 251 } 252 EXPORT_SYMBOL(mlx5_core_create_rqt); 253 254 int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in, 255 int inlen) 256 { 257 u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {}; 258 259 MLX5_SET(modify_rqt_in, in, rqtn, rqtn); 260 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT); 261 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 262 } 263 264 void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn) 265 { 266 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {}; 267 268 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT); 269 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn); 270 mlx5_cmd_exec_in(dev, destroy_rqt, in); 271 } 272 EXPORT_SYMBOL(mlx5_core_destroy_rqt); 273 274 static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev, 275 struct mlx5_hairpin_params *params, u32 *rqn) 276 { 277 u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0}; 278 void *rqc, *wq; 279 280 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 281 wq = MLX5_ADDR_OF(rqc, rqc, wq); 282 283 MLX5_SET(rqc, rqc, hairpin, 1); 284 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 285 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter); 286 287 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size); 288 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets); 289 290 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn); 291 } 292 293 static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev, 294 struct mlx5_hairpin_params *params, u32 *sqn) 295 { 296 u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0}; 297 void *sqc, *wq; 298 299 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 300 wq = MLX5_ADDR_OF(sqc, sqc, wq); 301 302 MLX5_SET(sqc, sqc, hairpin, 1); 303 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 304 305 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size); 306 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets); 307 308 return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn); 309 } 310 311 static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp, 312 struct mlx5_hairpin_params *params) 313 { 314 int i, j, err; 315 316 for (i = 0; i < hp->num_channels; i++) { 317 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]); 318 if (err) 319 goto out_err_rq; 320 } 321 322 for (i = 0; i < hp->num_channels; i++) { 323 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]); 324 if (err) 325 goto out_err_sq; 326 } 327 328 return 0; 329 330 out_err_sq: 331 for (j = 0; j < i; j++) 332 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]); 333 i = hp->num_channels; 334 out_err_rq: 335 for (j = 0; j < i; j++) 336 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]); 337 return err; 338 } 339 340 static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp) 341 { 342 int i; 343 344 for (i = 0; i < hp->num_channels; i++) { 345 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]); 346 if (!hp->peer_gone) 347 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]); 348 } 349 } 350 351 static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn, 352 int curr_state, int next_state, 353 u16 peer_vhca, u32 peer_sq) 354 { 355 u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {}; 356 void *rqc; 357 358 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 359 360 if (next_state == MLX5_RQC_STATE_RDY) { 361 MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq); 362 MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca); 363 } 364 365 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 366 MLX5_SET(rqc, rqc, state, next_state); 367 368 return mlx5_core_modify_rq(func_mdev, rqn, in); 369 } 370 371 static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn, 372 int curr_state, int next_state, 373 u16 peer_vhca, u32 peer_rq) 374 { 375 u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0}; 376 void *sqc; 377 378 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 379 380 if (next_state == MLX5_SQC_STATE_RDY) { 381 MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq); 382 MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca); 383 } 384 385 MLX5_SET(modify_sq_in, in, sq_state, curr_state); 386 MLX5_SET(sqc, sqc, state, next_state); 387 388 return mlx5_core_modify_sq(peer_mdev, sqn, in); 389 } 390 391 static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp) 392 { 393 int i, j, err; 394 395 /* set peer SQs */ 396 for (i = 0; i < hp->num_channels; i++) { 397 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], 398 MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY, 399 MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]); 400 if (err) 401 goto err_modify_sq; 402 } 403 404 /* set func RQs */ 405 for (i = 0; i < hp->num_channels; i++) { 406 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], 407 MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY, 408 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]); 409 if (err) 410 goto err_modify_rq; 411 } 412 413 return 0; 414 415 err_modify_rq: 416 for (j = 0; j < i; j++) 417 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY, 418 MLX5_RQC_STATE_RST, 0, 0); 419 i = hp->num_channels; 420 err_modify_sq: 421 for (j = 0; j < i; j++) 422 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY, 423 MLX5_SQC_STATE_RST, 0, 0); 424 return err; 425 } 426 427 static void mlx5_hairpin_unpair_peer_sq(struct mlx5_hairpin *hp) 428 { 429 int i; 430 431 for (i = 0; i < hp->num_channels; i++) 432 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY, 433 MLX5_SQC_STATE_RST, 0, 0); 434 } 435 436 static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp) 437 { 438 int i; 439 440 /* unset func RQs */ 441 for (i = 0; i < hp->num_channels; i++) 442 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY, 443 MLX5_RQC_STATE_RST, 0, 0); 444 /* unset peer SQs */ 445 if (!hp->peer_gone) 446 mlx5_hairpin_unpair_peer_sq(hp); 447 } 448 449 struct mlx5_hairpin * 450 mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev, 451 struct mlx5_core_dev *peer_mdev, 452 struct mlx5_hairpin_params *params) 453 { 454 struct mlx5_hairpin *hp; 455 int size, err; 456 457 size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32); 458 hp = kzalloc(size, GFP_KERNEL); 459 if (!hp) 460 return ERR_PTR(-ENOMEM); 461 462 hp->func_mdev = func_mdev; 463 hp->peer_mdev = peer_mdev; 464 hp->num_channels = params->num_channels; 465 466 hp->rqn = (void *)hp + sizeof(*hp); 467 hp->sqn = hp->rqn + params->num_channels; 468 469 /* alloc and pair func --> peer hairpin */ 470 err = mlx5_hairpin_create_queues(hp, params); 471 if (err) 472 goto err_create_queues; 473 474 err = mlx5_hairpin_pair_queues(hp); 475 if (err) 476 goto err_pair_queues; 477 478 return hp; 479 480 err_pair_queues: 481 mlx5_hairpin_destroy_queues(hp); 482 err_create_queues: 483 kfree(hp); 484 return ERR_PTR(err); 485 } 486 487 void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp) 488 { 489 mlx5_hairpin_unpair_queues(hp); 490 mlx5_hairpin_destroy_queues(hp); 491 kfree(hp); 492 } 493 494 void mlx5_core_hairpin_clear_dead_peer(struct mlx5_hairpin *hp) 495 { 496 int i; 497 498 mlx5_hairpin_unpair_peer_sq(hp); 499 500 /* destroy peer SQ */ 501 for (i = 0; i < hp->num_channels; i++) 502 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]); 503 504 hp->peer_gone = true; 505 } 506