1 /* 2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #define CREATE_TRACE_POINTS 33 #include "lib/eq.h" 34 #include "fw_tracer.h" 35 #include "fw_tracer_tracepoint.h" 36 37 static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer) 38 { 39 u32 *string_db_base_address_out = tracer->str_db.base_address_out; 40 u32 *string_db_size_out = tracer->str_db.size_out; 41 struct mlx5_core_dev *dev = tracer->dev; 42 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 43 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 44 void *mtrc_cap_sp; 45 int err, i; 46 47 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 48 MLX5_REG_MTRC_CAP, 0, 0); 49 if (err) { 50 mlx5_core_warn(dev, "FWTracer: Error reading tracer caps %d\n", 51 err); 52 return err; 53 } 54 55 if (!MLX5_GET(mtrc_cap, out, trace_to_memory)) { 56 mlx5_core_dbg(dev, "FWTracer: Device does not support logging traces to memory\n"); 57 return -ENOTSUPP; 58 } 59 60 tracer->trc_ver = MLX5_GET(mtrc_cap, out, trc_ver); 61 tracer->str_db.first_string_trace = 62 MLX5_GET(mtrc_cap, out, first_string_trace); 63 tracer->str_db.num_string_trace = 64 MLX5_GET(mtrc_cap, out, num_string_trace); 65 tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db); 66 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner); 67 68 for (i = 0; i < tracer->str_db.num_string_db; i++) { 69 mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]); 70 string_db_base_address_out[i] = MLX5_GET(mtrc_string_db_param, 71 mtrc_cap_sp, 72 string_db_base_address); 73 string_db_size_out[i] = MLX5_GET(mtrc_string_db_param, 74 mtrc_cap_sp, string_db_size); 75 } 76 77 return err; 78 } 79 80 static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer *tracer, 81 u32 *out, u32 out_size, 82 u8 trace_owner) 83 { 84 struct mlx5_core_dev *dev = tracer->dev; 85 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 86 87 MLX5_SET(mtrc_cap, in, trace_owner, trace_owner); 88 89 return mlx5_core_access_reg(dev, in, sizeof(in), out, out_size, 90 MLX5_REG_MTRC_CAP, 0, 1); 91 } 92 93 static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer *tracer) 94 { 95 struct mlx5_core_dev *dev = tracer->dev; 96 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 97 int err; 98 99 err = mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out), 100 MLX5_FW_TRACER_ACQUIRE_OWNERSHIP); 101 if (err) { 102 mlx5_core_warn(dev, "FWTracer: Acquire tracer ownership failed %d\n", 103 err); 104 return err; 105 } 106 107 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner); 108 109 if (!tracer->owner) 110 return -EBUSY; 111 112 return 0; 113 } 114 115 static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer) 116 { 117 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 118 119 mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out), 120 MLX5_FW_TRACER_RELEASE_OWNERSHIP); 121 tracer->owner = false; 122 } 123 124 static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer) 125 { 126 struct mlx5_core_dev *dev = tracer->dev; 127 struct device *ddev; 128 dma_addr_t dma; 129 void *buff; 130 gfp_t gfp; 131 int err; 132 133 tracer->buff.size = TRACE_BUFFER_SIZE_BYTE; 134 135 gfp = GFP_KERNEL | __GFP_ZERO; 136 buff = (void *)__get_free_pages(gfp, 137 get_order(tracer->buff.size)); 138 if (!buff) { 139 err = -ENOMEM; 140 mlx5_core_warn(dev, "FWTracer: Failed to allocate pages, %d\n", err); 141 return err; 142 } 143 tracer->buff.log_buf = buff; 144 145 ddev = mlx5_core_dma_dev(dev); 146 dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE); 147 if (dma_mapping_error(ddev, dma)) { 148 mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n", 149 dma_mapping_error(ddev, dma)); 150 err = -ENOMEM; 151 goto free_pages; 152 } 153 tracer->buff.dma = dma; 154 155 return 0; 156 157 free_pages: 158 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size)); 159 160 return err; 161 } 162 163 static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer) 164 { 165 struct mlx5_core_dev *dev = tracer->dev; 166 struct device *ddev; 167 168 if (!tracer->buff.log_buf) 169 return; 170 171 ddev = mlx5_core_dma_dev(dev); 172 dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE); 173 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size)); 174 } 175 176 static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer *tracer) 177 { 178 struct mlx5_core_dev *dev = tracer->dev; 179 int err, inlen, i; 180 __be64 *mtt; 181 void *mkc; 182 u32 *in; 183 184 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + 185 sizeof(*mtt) * round_up(TRACER_BUFFER_PAGE_NUM, 2); 186 187 in = kvzalloc(inlen, GFP_KERNEL); 188 if (!in) 189 return -ENOMEM; 190 191 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 192 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2)); 193 mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 194 for (i = 0 ; i < TRACER_BUFFER_PAGE_NUM ; i++) 195 mtt[i] = cpu_to_be64(tracer->buff.dma + i * PAGE_SIZE); 196 197 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 198 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 199 MLX5_SET(mkc, mkc, lr, 1); 200 MLX5_SET(mkc, mkc, lw, 1); 201 MLX5_SET(mkc, mkc, pd, tracer->buff.pdn); 202 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 203 MLX5_SET(mkc, mkc, qpn, 0xffffff); 204 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 205 MLX5_SET(mkc, mkc, translations_octword_size, 206 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2)); 207 MLX5_SET64(mkc, mkc, start_addr, tracer->buff.dma); 208 MLX5_SET64(mkc, mkc, len, tracer->buff.size); 209 err = mlx5_core_create_mkey(dev, &tracer->buff.mkey, in, inlen); 210 if (err) 211 mlx5_core_warn(dev, "FWTracer: Failed to create mkey, %d\n", err); 212 213 kvfree(in); 214 215 return err; 216 } 217 218 static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer *tracer) 219 { 220 u32 num_string_db = tracer->str_db.num_string_db; 221 int i; 222 223 for (i = 0; i < num_string_db; i++) { 224 kfree(tracer->str_db.buffer[i]); 225 tracer->str_db.buffer[i] = NULL; 226 } 227 } 228 229 static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer) 230 { 231 u32 *string_db_size_out = tracer->str_db.size_out; 232 u32 num_string_db = tracer->str_db.num_string_db; 233 int i; 234 235 for (i = 0; i < num_string_db; i++) { 236 tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL); 237 if (!tracer->str_db.buffer[i]) 238 goto free_strings_db; 239 } 240 241 return 0; 242 243 free_strings_db: 244 mlx5_fw_tracer_free_strings_db(tracer); 245 return -ENOMEM; 246 } 247 248 static void 249 mlx5_fw_tracer_init_saved_traces_array(struct mlx5_fw_tracer *tracer) 250 { 251 tracer->st_arr.saved_traces_index = 0; 252 mutex_init(&tracer->st_arr.lock); 253 } 254 255 static void 256 mlx5_fw_tracer_clean_saved_traces_array(struct mlx5_fw_tracer *tracer) 257 { 258 mutex_destroy(&tracer->st_arr.lock); 259 } 260 261 static void mlx5_tracer_read_strings_db(struct work_struct *work) 262 { 263 struct mlx5_fw_tracer *tracer = container_of(work, struct mlx5_fw_tracer, 264 read_fw_strings_work); 265 u32 num_of_reads, num_string_db = tracer->str_db.num_string_db; 266 struct mlx5_core_dev *dev = tracer->dev; 267 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 268 u32 leftovers, offset; 269 int err = 0, i, j; 270 u32 *out, outlen; 271 void *out_value; 272 273 outlen = MLX5_ST_SZ_BYTES(mtrc_stdb) + STRINGS_DB_READ_SIZE_BYTES; 274 out = kzalloc(outlen, GFP_KERNEL); 275 if (!out) { 276 err = -ENOMEM; 277 goto out; 278 } 279 280 for (i = 0; i < num_string_db; i++) { 281 offset = 0; 282 MLX5_SET(mtrc_stdb, in, string_db_index, i); 283 num_of_reads = tracer->str_db.size_out[i] / 284 STRINGS_DB_READ_SIZE_BYTES; 285 leftovers = (tracer->str_db.size_out[i] % 286 STRINGS_DB_READ_SIZE_BYTES) / 287 STRINGS_DB_LEFTOVER_SIZE_BYTES; 288 289 MLX5_SET(mtrc_stdb, in, read_size, STRINGS_DB_READ_SIZE_BYTES); 290 for (j = 0; j < num_of_reads; j++) { 291 MLX5_SET(mtrc_stdb, in, start_offset, offset); 292 293 err = mlx5_core_access_reg(dev, in, sizeof(in), out, 294 outlen, MLX5_REG_MTRC_STDB, 295 0, 1); 296 if (err) { 297 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n", 298 err); 299 goto out_free; 300 } 301 302 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data); 303 memcpy(tracer->str_db.buffer[i] + offset, out_value, 304 STRINGS_DB_READ_SIZE_BYTES); 305 offset += STRINGS_DB_READ_SIZE_BYTES; 306 } 307 308 /* Strings database is aligned to 64, need to read leftovers*/ 309 MLX5_SET(mtrc_stdb, in, read_size, 310 STRINGS_DB_LEFTOVER_SIZE_BYTES); 311 for (j = 0; j < leftovers; j++) { 312 MLX5_SET(mtrc_stdb, in, start_offset, offset); 313 314 err = mlx5_core_access_reg(dev, in, sizeof(in), out, 315 outlen, MLX5_REG_MTRC_STDB, 316 0, 1); 317 if (err) { 318 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n", 319 err); 320 goto out_free; 321 } 322 323 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data); 324 memcpy(tracer->str_db.buffer[i] + offset, out_value, 325 STRINGS_DB_LEFTOVER_SIZE_BYTES); 326 offset += STRINGS_DB_LEFTOVER_SIZE_BYTES; 327 } 328 } 329 330 tracer->str_db.loaded = true; 331 332 out_free: 333 kfree(out); 334 out: 335 return; 336 } 337 338 static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev) 339 { 340 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 341 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 342 int err; 343 344 MLX5_SET(mtrc_ctrl, in, arm_event, 1); 345 346 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 347 MLX5_REG_MTRC_CTRL, 0, 1); 348 if (err) 349 mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err); 350 } 351 352 static const char *VAL_PARM = "%llx"; 353 static const char *REPLACE_64_VAL_PARM = "%x%x"; 354 static const char *PARAM_CHAR = "%"; 355 356 static int mlx5_tracer_message_hash(u32 message_id) 357 { 358 return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1); 359 } 360 361 static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer, 362 struct tracer_event *tracer_event) 363 { 364 struct hlist_head *head = 365 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)]; 366 struct tracer_string_format *cur_string; 367 368 cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL); 369 if (!cur_string) 370 return NULL; 371 372 hlist_add_head(&cur_string->hlist, head); 373 374 return cur_string; 375 } 376 377 static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer, 378 struct tracer_event *tracer_event) 379 { 380 struct tracer_string_format *cur_string; 381 u32 str_ptr, offset; 382 int i; 383 384 str_ptr = tracer_event->string_event.string_param; 385 386 for (i = 0; i < tracer->str_db.num_string_db; i++) { 387 if (str_ptr > tracer->str_db.base_address_out[i] && 388 str_ptr < tracer->str_db.base_address_out[i] + 389 tracer->str_db.size_out[i]) { 390 offset = str_ptr - tracer->str_db.base_address_out[i]; 391 /* add it to the hash */ 392 cur_string = mlx5_tracer_message_insert(tracer, tracer_event); 393 if (!cur_string) 394 return NULL; 395 cur_string->string = (char *)(tracer->str_db.buffer[i] + 396 offset); 397 return cur_string; 398 } 399 } 400 401 return NULL; 402 } 403 404 static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt) 405 { 406 hlist_del(&str_frmt->hlist); 407 kfree(str_frmt); 408 } 409 410 static int mlx5_tracer_get_num_of_params(char *str) 411 { 412 char *substr, *pstr = str; 413 int num_of_params = 0; 414 415 /* replace %llx with %x%x */ 416 substr = strstr(pstr, VAL_PARM); 417 while (substr) { 418 memcpy(substr, REPLACE_64_VAL_PARM, 4); 419 pstr = substr; 420 substr = strstr(pstr, VAL_PARM); 421 } 422 423 /* count all the % characters */ 424 substr = strstr(str, PARAM_CHAR); 425 while (substr) { 426 num_of_params += 1; 427 str = substr + 1; 428 substr = strstr(str, PARAM_CHAR); 429 } 430 431 return num_of_params; 432 } 433 434 static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head, 435 u8 event_id, u32 tmsn) 436 { 437 struct tracer_string_format *message; 438 439 hlist_for_each_entry(message, head, hlist) 440 if (message->event_id == event_id && message->tmsn == tmsn) 441 return message; 442 443 return NULL; 444 } 445 446 static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer, 447 struct tracer_event *tracer_event) 448 { 449 struct hlist_head *head = 450 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)]; 451 452 return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn); 453 } 454 455 static void poll_trace(struct mlx5_fw_tracer *tracer, 456 struct tracer_event *tracer_event, u64 *trace) 457 { 458 u32 timestamp_low, timestamp_mid, timestamp_high, urts; 459 460 tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id); 461 tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost); 462 463 switch (tracer_event->event_id) { 464 case TRACER_EVENT_TYPE_TIMESTAMP: 465 tracer_event->type = TRACER_EVENT_TYPE_TIMESTAMP; 466 urts = MLX5_GET(tracer_timestamp_event, trace, urts); 467 if (tracer->trc_ver == 0) 468 tracer_event->timestamp_event.unreliable = !!(urts >> 2); 469 else 470 tracer_event->timestamp_event.unreliable = !!(urts & 1); 471 472 timestamp_low = MLX5_GET(tracer_timestamp_event, 473 trace, timestamp7_0); 474 timestamp_mid = MLX5_GET(tracer_timestamp_event, 475 trace, timestamp39_8); 476 timestamp_high = MLX5_GET(tracer_timestamp_event, 477 trace, timestamp52_40); 478 479 tracer_event->timestamp_event.timestamp = 480 ((u64)timestamp_high << 40) | 481 ((u64)timestamp_mid << 8) | 482 (u64)timestamp_low; 483 break; 484 default: 485 if (tracer_event->event_id >= tracer->str_db.first_string_trace || 486 tracer_event->event_id <= tracer->str_db.first_string_trace + 487 tracer->str_db.num_string_trace) { 488 tracer_event->type = TRACER_EVENT_TYPE_STRING; 489 tracer_event->string_event.timestamp = 490 MLX5_GET(tracer_string_event, trace, timestamp); 491 tracer_event->string_event.string_param = 492 MLX5_GET(tracer_string_event, trace, string_param); 493 tracer_event->string_event.tmsn = 494 MLX5_GET(tracer_string_event, trace, tmsn); 495 tracer_event->string_event.tdsn = 496 MLX5_GET(tracer_string_event, trace, tdsn); 497 } else { 498 tracer_event->type = TRACER_EVENT_TYPE_UNRECOGNIZED; 499 } 500 break; 501 } 502 } 503 504 static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event) 505 { 506 struct tracer_event tracer_event; 507 u8 event_id; 508 509 event_id = MLX5_GET(tracer_event, ts_event, event_id); 510 511 if (event_id == TRACER_EVENT_TYPE_TIMESTAMP) 512 poll_trace(tracer, &tracer_event, ts_event); 513 else 514 tracer_event.timestamp_event.timestamp = 0; 515 516 return tracer_event.timestamp_event.timestamp; 517 } 518 519 static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer) 520 { 521 struct tracer_string_format *str_frmt; 522 struct hlist_node *n; 523 int i; 524 525 for (i = 0; i < MESSAGE_HASH_SIZE; i++) { 526 hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist) 527 mlx5_tracer_clean_message(str_frmt); 528 } 529 } 530 531 static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer) 532 { 533 struct tracer_string_format *str_frmt, *tmp_str; 534 535 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, 536 list) 537 list_del(&str_frmt->list); 538 } 539 540 static void mlx5_fw_tracer_save_trace(struct mlx5_fw_tracer *tracer, 541 u64 timestamp, bool lost, 542 u8 event_id, char *msg) 543 { 544 struct mlx5_fw_trace_data *trace_data; 545 546 mutex_lock(&tracer->st_arr.lock); 547 trace_data = &tracer->st_arr.straces[tracer->st_arr.saved_traces_index]; 548 trace_data->timestamp = timestamp; 549 trace_data->lost = lost; 550 trace_data->event_id = event_id; 551 strscpy_pad(trace_data->msg, msg, TRACE_STR_MSG); 552 553 tracer->st_arr.saved_traces_index = 554 (tracer->st_arr.saved_traces_index + 1) & (SAVED_TRACES_NUM - 1); 555 mutex_unlock(&tracer->st_arr.lock); 556 } 557 558 static noinline 559 void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt, 560 struct mlx5_core_dev *dev, 561 u64 trace_timestamp) 562 { 563 char tmp[512]; 564 565 snprintf(tmp, sizeof(tmp), str_frmt->string, 566 str_frmt->params[0], 567 str_frmt->params[1], 568 str_frmt->params[2], 569 str_frmt->params[3], 570 str_frmt->params[4], 571 str_frmt->params[5], 572 str_frmt->params[6]); 573 574 trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost, 575 str_frmt->event_id, tmp); 576 577 mlx5_fw_tracer_save_trace(dev->tracer, trace_timestamp, 578 str_frmt->lost, str_frmt->event_id, tmp); 579 580 /* remove it from hash */ 581 mlx5_tracer_clean_message(str_frmt); 582 } 583 584 static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, 585 struct tracer_event *tracer_event) 586 { 587 struct tracer_string_format *cur_string; 588 589 if (tracer_event->string_event.tdsn == 0) { 590 cur_string = mlx5_tracer_get_string(tracer, tracer_event); 591 if (!cur_string) 592 return -1; 593 594 cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string); 595 cur_string->last_param_num = 0; 596 cur_string->event_id = tracer_event->event_id; 597 cur_string->tmsn = tracer_event->string_event.tmsn; 598 cur_string->timestamp = tracer_event->string_event.timestamp; 599 cur_string->lost = tracer_event->lost_event; 600 if (cur_string->num_of_params == 0) /* trace with no params */ 601 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 602 } else { 603 cur_string = mlx5_tracer_message_get(tracer, tracer_event); 604 if (!cur_string) { 605 pr_debug("%s Got string event for unknown string tdsm: %d\n", 606 __func__, tracer_event->string_event.tmsn); 607 return -1; 608 } 609 cur_string->last_param_num += 1; 610 if (cur_string->last_param_num > TRACER_MAX_PARAMS) { 611 pr_debug("%s Number of params exceeds the max (%d)\n", 612 __func__, TRACER_MAX_PARAMS); 613 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 614 return 0; 615 } 616 /* keep the new parameter */ 617 cur_string->params[cur_string->last_param_num - 1] = 618 tracer_event->string_event.string_param; 619 if (cur_string->last_param_num == cur_string->num_of_params) 620 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 621 } 622 623 return 0; 624 } 625 626 static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer, 627 struct tracer_event *tracer_event) 628 { 629 struct tracer_timestamp_event timestamp_event = 630 tracer_event->timestamp_event; 631 struct tracer_string_format *str_frmt, *tmp_str; 632 struct mlx5_core_dev *dev = tracer->dev; 633 u64 trace_timestamp; 634 635 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) { 636 list_del(&str_frmt->list); 637 if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0)) 638 trace_timestamp = (timestamp_event.timestamp & MASK_52_7) | 639 (str_frmt->timestamp & MASK_6_0); 640 else 641 trace_timestamp = ((timestamp_event.timestamp & MASK_52_7) - 1) | 642 (str_frmt->timestamp & MASK_6_0); 643 644 mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp); 645 } 646 } 647 648 static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer, 649 struct tracer_event *tracer_event) 650 { 651 if (tracer_event->type == TRACER_EVENT_TYPE_STRING) { 652 mlx5_tracer_handle_string_trace(tracer, tracer_event); 653 } else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) { 654 if (!tracer_event->timestamp_event.unreliable) 655 mlx5_tracer_handle_timestamp_trace(tracer, tracer_event); 656 } else { 657 pr_debug("%s Got unrecognised type %d for parsing, exiting..\n", 658 __func__, tracer_event->type); 659 } 660 return 0; 661 } 662 663 static void mlx5_fw_tracer_handle_traces(struct work_struct *work) 664 { 665 struct mlx5_fw_tracer *tracer = 666 container_of(work, struct mlx5_fw_tracer, handle_traces_work); 667 u64 block_timestamp, last_block_timestamp, tmp_trace_block[TRACES_PER_BLOCK]; 668 u32 block_count, start_offset, prev_start_offset, prev_consumer_index; 669 u32 trace_event_size = MLX5_ST_SZ_BYTES(tracer_event); 670 struct mlx5_core_dev *dev = tracer->dev; 671 struct tracer_event tracer_event; 672 int i; 673 674 mlx5_core_dbg(dev, "FWTracer: Handle Trace event, owner=(%d)\n", tracer->owner); 675 if (!tracer->owner) 676 return; 677 678 block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE; 679 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE; 680 681 /* Copy the block to local buffer to avoid HW override while being processed */ 682 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset, 683 TRACER_BLOCK_SIZE_BYTE); 684 685 block_timestamp = 686 get_block_timestamp(tracer, &tmp_trace_block[TRACES_PER_BLOCK - 1]); 687 688 while (block_timestamp > tracer->last_timestamp) { 689 /* Check block override if it's not the first block */ 690 if (!tracer->last_timestamp) { 691 u64 *ts_event; 692 /* To avoid block override be the HW in case of buffer 693 * wraparound, the time stamp of the previous block 694 * should be compared to the last timestamp handled 695 * by the driver. 696 */ 697 prev_consumer_index = 698 (tracer->buff.consumer_index - 1) & (block_count - 1); 699 prev_start_offset = prev_consumer_index * TRACER_BLOCK_SIZE_BYTE; 700 701 ts_event = tracer->buff.log_buf + prev_start_offset + 702 (TRACES_PER_BLOCK - 1) * trace_event_size; 703 last_block_timestamp = get_block_timestamp(tracer, ts_event); 704 /* If previous timestamp different from last stored 705 * timestamp then there is a good chance that the 706 * current buffer is overwritten and therefore should 707 * not be parsed. 708 */ 709 if (tracer->last_timestamp != last_block_timestamp) { 710 mlx5_core_warn(dev, "FWTracer: Events were lost\n"); 711 tracer->last_timestamp = block_timestamp; 712 tracer->buff.consumer_index = 713 (tracer->buff.consumer_index + 1) & (block_count - 1); 714 break; 715 } 716 } 717 718 /* Parse events */ 719 for (i = 0; i < TRACES_PER_BLOCK ; i++) { 720 poll_trace(tracer, &tracer_event, &tmp_trace_block[i]); 721 mlx5_tracer_handle_trace(tracer, &tracer_event); 722 } 723 724 tracer->buff.consumer_index = 725 (tracer->buff.consumer_index + 1) & (block_count - 1); 726 727 tracer->last_timestamp = block_timestamp; 728 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE; 729 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset, 730 TRACER_BLOCK_SIZE_BYTE); 731 block_timestamp = get_block_timestamp(tracer, 732 &tmp_trace_block[TRACES_PER_BLOCK - 1]); 733 } 734 735 mlx5_fw_tracer_arm(dev); 736 } 737 738 static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer) 739 { 740 struct mlx5_core_dev *dev = tracer->dev; 741 u32 out[MLX5_ST_SZ_DW(mtrc_conf)] = {0}; 742 u32 in[MLX5_ST_SZ_DW(mtrc_conf)] = {0}; 743 int err; 744 745 MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY); 746 MLX5_SET(mtrc_conf, in, log_trace_buffer_size, 747 ilog2(TRACER_BUFFER_PAGE_NUM)); 748 MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey); 749 750 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 751 MLX5_REG_MTRC_CONF, 0, 1); 752 if (err) 753 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err); 754 755 return err; 756 } 757 758 static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer *tracer, u8 status, u8 arm) 759 { 760 struct mlx5_core_dev *dev = tracer->dev; 761 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 762 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 763 int err; 764 765 MLX5_SET(mtrc_ctrl, in, modify_field_select, TRACE_STATUS); 766 MLX5_SET(mtrc_ctrl, in, trace_status, status); 767 MLX5_SET(mtrc_ctrl, in, arm_event, arm); 768 769 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 770 MLX5_REG_MTRC_CTRL, 0, 1); 771 772 if (!err && status) 773 tracer->last_timestamp = 0; 774 775 return err; 776 } 777 778 static int mlx5_fw_tracer_start(struct mlx5_fw_tracer *tracer) 779 { 780 struct mlx5_core_dev *dev = tracer->dev; 781 int err; 782 783 err = mlx5_fw_tracer_ownership_acquire(tracer); 784 if (err) { 785 mlx5_core_dbg(dev, "FWTracer: Ownership was not granted %d\n", err); 786 /* Don't fail since ownership can be acquired on a later FW event */ 787 return 0; 788 } 789 790 err = mlx5_fw_tracer_set_mtrc_conf(tracer); 791 if (err) { 792 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configuration %d\n", err); 793 goto release_ownership; 794 } 795 796 /* enable tracer & trace events */ 797 err = mlx5_fw_tracer_set_mtrc_ctrl(tracer, 1, 1); 798 if (err) { 799 mlx5_core_warn(dev, "FWTracer: Failed to enable tracer %d\n", err); 800 goto release_ownership; 801 } 802 803 mlx5_core_dbg(dev, "FWTracer: Ownership granted and active\n"); 804 return 0; 805 806 release_ownership: 807 mlx5_fw_tracer_ownership_release(tracer); 808 return err; 809 } 810 811 static void mlx5_fw_tracer_ownership_change(struct work_struct *work) 812 { 813 struct mlx5_fw_tracer *tracer = 814 container_of(work, struct mlx5_fw_tracer, ownership_change_work); 815 816 mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner); 817 if (tracer->owner) { 818 tracer->owner = false; 819 tracer->buff.consumer_index = 0; 820 return; 821 } 822 823 mlx5_fw_tracer_start(tracer); 824 } 825 826 static int mlx5_fw_tracer_set_core_dump_reg(struct mlx5_core_dev *dev, 827 u32 *in, int size_in) 828 { 829 u32 out[MLX5_ST_SZ_DW(core_dump_reg)] = {}; 830 831 if (!MLX5_CAP_DEBUG(dev, core_dump_general) && 832 !MLX5_CAP_DEBUG(dev, core_dump_qp)) 833 return -EOPNOTSUPP; 834 835 return mlx5_core_access_reg(dev, in, size_in, out, sizeof(out), 836 MLX5_REG_CORE_DUMP, 0, 1); 837 } 838 839 int mlx5_fw_tracer_trigger_core_dump_general(struct mlx5_core_dev *dev) 840 { 841 struct mlx5_fw_tracer *tracer = dev->tracer; 842 u32 in[MLX5_ST_SZ_DW(core_dump_reg)] = {}; 843 int err; 844 845 if (!MLX5_CAP_DEBUG(dev, core_dump_general) || !tracer) 846 return -EOPNOTSUPP; 847 if (!tracer->owner) 848 return -EPERM; 849 850 MLX5_SET(core_dump_reg, in, core_dump_type, 0x0); 851 852 err = mlx5_fw_tracer_set_core_dump_reg(dev, in, sizeof(in)); 853 if (err) 854 return err; 855 queue_work(tracer->work_queue, &tracer->handle_traces_work); 856 flush_workqueue(tracer->work_queue); 857 return 0; 858 } 859 860 static int 861 mlx5_devlink_fmsg_fill_trace(struct devlink_fmsg *fmsg, 862 struct mlx5_fw_trace_data *trace_data) 863 { 864 int err; 865 866 err = devlink_fmsg_obj_nest_start(fmsg); 867 if (err) 868 return err; 869 870 err = devlink_fmsg_u64_pair_put(fmsg, "timestamp", trace_data->timestamp); 871 if (err) 872 return err; 873 874 err = devlink_fmsg_bool_pair_put(fmsg, "lost", trace_data->lost); 875 if (err) 876 return err; 877 878 err = devlink_fmsg_u8_pair_put(fmsg, "event_id", trace_data->event_id); 879 if (err) 880 return err; 881 882 err = devlink_fmsg_string_pair_put(fmsg, "msg", trace_data->msg); 883 if (err) 884 return err; 885 886 err = devlink_fmsg_obj_nest_end(fmsg); 887 if (err) 888 return err; 889 return 0; 890 } 891 892 int mlx5_fw_tracer_get_saved_traces_objects(struct mlx5_fw_tracer *tracer, 893 struct devlink_fmsg *fmsg) 894 { 895 struct mlx5_fw_trace_data *straces = tracer->st_arr.straces; 896 u32 index, start_index, end_index; 897 u32 saved_traces_index; 898 int err; 899 900 if (!straces[0].timestamp) 901 return -ENOMSG; 902 903 mutex_lock(&tracer->st_arr.lock); 904 saved_traces_index = tracer->st_arr.saved_traces_index; 905 if (straces[saved_traces_index].timestamp) 906 start_index = saved_traces_index; 907 else 908 start_index = 0; 909 end_index = (saved_traces_index - 1) & (SAVED_TRACES_NUM - 1); 910 911 err = devlink_fmsg_arr_pair_nest_start(fmsg, "dump fw traces"); 912 if (err) 913 goto unlock; 914 index = start_index; 915 while (index != end_index) { 916 err = mlx5_devlink_fmsg_fill_trace(fmsg, &straces[index]); 917 if (err) 918 goto unlock; 919 920 index = (index + 1) & (SAVED_TRACES_NUM - 1); 921 } 922 923 err = devlink_fmsg_arr_pair_nest_end(fmsg); 924 unlock: 925 mutex_unlock(&tracer->st_arr.lock); 926 return err; 927 } 928 929 /* Create software resources (Buffers, etc ..) */ 930 struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev) 931 { 932 struct mlx5_fw_tracer *tracer = NULL; 933 int err; 934 935 if (!MLX5_CAP_MCAM_REG(dev, tracer_registers)) { 936 mlx5_core_dbg(dev, "FWTracer: Tracer capability not present\n"); 937 return NULL; 938 } 939 940 tracer = kvzalloc(sizeof(*tracer), GFP_KERNEL); 941 if (!tracer) 942 return ERR_PTR(-ENOMEM); 943 944 tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer"); 945 if (!tracer->work_queue) { 946 err = -ENOMEM; 947 goto free_tracer; 948 } 949 950 tracer->dev = dev; 951 952 INIT_LIST_HEAD(&tracer->ready_strings_list); 953 INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change); 954 INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db); 955 INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces); 956 957 958 err = mlx5_query_mtrc_caps(tracer); 959 if (err) { 960 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err); 961 goto destroy_workqueue; 962 } 963 964 err = mlx5_fw_tracer_create_log_buf(tracer); 965 if (err) { 966 mlx5_core_warn(dev, "FWTracer: Create log buffer failed %d\n", err); 967 goto destroy_workqueue; 968 } 969 970 err = mlx5_fw_tracer_allocate_strings_db(tracer); 971 if (err) { 972 mlx5_core_warn(dev, "FWTracer: Allocate strings database failed %d\n", err); 973 goto free_log_buf; 974 } 975 976 mlx5_fw_tracer_init_saved_traces_array(tracer); 977 mlx5_core_dbg(dev, "FWTracer: Tracer created\n"); 978 979 return tracer; 980 981 free_log_buf: 982 mlx5_fw_tracer_destroy_log_buf(tracer); 983 destroy_workqueue: 984 tracer->dev = NULL; 985 destroy_workqueue(tracer->work_queue); 986 free_tracer: 987 kvfree(tracer); 988 return ERR_PTR(err); 989 } 990 991 static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data); 992 993 /* Create HW resources + start tracer */ 994 int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer) 995 { 996 struct mlx5_core_dev *dev; 997 int err; 998 999 if (IS_ERR_OR_NULL(tracer)) 1000 return 0; 1001 1002 dev = tracer->dev; 1003 1004 if (!tracer->str_db.loaded) 1005 queue_work(tracer->work_queue, &tracer->read_fw_strings_work); 1006 1007 err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn); 1008 if (err) { 1009 mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err); 1010 goto err_cancel_work; 1011 } 1012 1013 err = mlx5_fw_tracer_create_mkey(tracer); 1014 if (err) { 1015 mlx5_core_warn(dev, "FWTracer: Failed to create mkey %d\n", err); 1016 goto err_dealloc_pd; 1017 } 1018 1019 MLX5_NB_INIT(&tracer->nb, fw_tracer_event, DEVICE_TRACER); 1020 mlx5_eq_notifier_register(dev, &tracer->nb); 1021 1022 err = mlx5_fw_tracer_start(tracer); 1023 if (err) { 1024 mlx5_core_warn(dev, "FWTracer: Failed to start tracer %d\n", err); 1025 goto err_notifier_unregister; 1026 } 1027 return 0; 1028 1029 err_notifier_unregister: 1030 mlx5_eq_notifier_unregister(dev, &tracer->nb); 1031 mlx5_core_destroy_mkey(dev, tracer->buff.mkey); 1032 err_dealloc_pd: 1033 mlx5_core_dealloc_pd(dev, tracer->buff.pdn); 1034 err_cancel_work: 1035 cancel_work_sync(&tracer->read_fw_strings_work); 1036 return err; 1037 } 1038 1039 /* Stop tracer + Cleanup HW resources */ 1040 void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer) 1041 { 1042 if (IS_ERR_OR_NULL(tracer)) 1043 return; 1044 1045 mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n", 1046 tracer->owner); 1047 mlx5_eq_notifier_unregister(tracer->dev, &tracer->nb); 1048 cancel_work_sync(&tracer->ownership_change_work); 1049 cancel_work_sync(&tracer->handle_traces_work); 1050 1051 if (tracer->owner) 1052 mlx5_fw_tracer_ownership_release(tracer); 1053 1054 mlx5_core_destroy_mkey(tracer->dev, tracer->buff.mkey); 1055 mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn); 1056 } 1057 1058 /* Free software resources (Buffers, etc ..) */ 1059 void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer) 1060 { 1061 if (IS_ERR_OR_NULL(tracer)) 1062 return; 1063 1064 mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n"); 1065 1066 cancel_work_sync(&tracer->read_fw_strings_work); 1067 mlx5_fw_tracer_clean_ready_list(tracer); 1068 mlx5_fw_tracer_clean_print_hash(tracer); 1069 mlx5_fw_tracer_clean_saved_traces_array(tracer); 1070 mlx5_fw_tracer_free_strings_db(tracer); 1071 mlx5_fw_tracer_destroy_log_buf(tracer); 1072 destroy_workqueue(tracer->work_queue); 1073 kvfree(tracer); 1074 } 1075 1076 static int mlx5_fw_tracer_recreate_strings_db(struct mlx5_fw_tracer *tracer) 1077 { 1078 struct mlx5_core_dev *dev; 1079 int err; 1080 1081 cancel_work_sync(&tracer->read_fw_strings_work); 1082 mlx5_fw_tracer_clean_ready_list(tracer); 1083 mlx5_fw_tracer_clean_print_hash(tracer); 1084 mlx5_fw_tracer_clean_saved_traces_array(tracer); 1085 mlx5_fw_tracer_free_strings_db(tracer); 1086 1087 dev = tracer->dev; 1088 err = mlx5_query_mtrc_caps(tracer); 1089 if (err) { 1090 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err); 1091 return err; 1092 } 1093 1094 err = mlx5_fw_tracer_allocate_strings_db(tracer); 1095 if (err) { 1096 mlx5_core_warn(dev, "FWTracer: Allocate strings DB failed %d\n", err); 1097 return err; 1098 } 1099 mlx5_fw_tracer_init_saved_traces_array(tracer); 1100 1101 return 0; 1102 } 1103 1104 int mlx5_fw_tracer_reload(struct mlx5_fw_tracer *tracer) 1105 { 1106 struct mlx5_core_dev *dev; 1107 int err; 1108 1109 if (IS_ERR_OR_NULL(tracer)) 1110 return 0; 1111 1112 dev = tracer->dev; 1113 mlx5_fw_tracer_cleanup(tracer); 1114 err = mlx5_fw_tracer_recreate_strings_db(tracer); 1115 if (err) { 1116 mlx5_core_warn(dev, "Failed to recreate FW tracer strings DB\n"); 1117 return err; 1118 } 1119 err = mlx5_fw_tracer_init(tracer); 1120 if (err) { 1121 mlx5_core_warn(dev, "Failed to re-initialize FW tracer\n"); 1122 return err; 1123 } 1124 1125 return 0; 1126 } 1127 1128 static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data) 1129 { 1130 struct mlx5_fw_tracer *tracer = mlx5_nb_cof(nb, struct mlx5_fw_tracer, nb); 1131 struct mlx5_core_dev *dev = tracer->dev; 1132 struct mlx5_eqe *eqe = data; 1133 1134 switch (eqe->sub_type) { 1135 case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE: 1136 queue_work(tracer->work_queue, &tracer->ownership_change_work); 1137 break; 1138 case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE: 1139 if (likely(tracer->str_db.loaded)) 1140 queue_work(tracer->work_queue, &tracer->handle_traces_work); 1141 break; 1142 default: 1143 mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n", 1144 eqe->sub_type); 1145 } 1146 1147 return NOTIFY_OK; 1148 } 1149 1150 EXPORT_TRACEPOINT_SYMBOL(mlx5_fw); 1151