1 /* 2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #define CREATE_TRACE_POINTS 33 #include "lib/eq.h" 34 #include "fw_tracer.h" 35 #include "fw_tracer_tracepoint.h" 36 37 static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer) 38 { 39 u32 *string_db_base_address_out = tracer->str_db.base_address_out; 40 u32 *string_db_size_out = tracer->str_db.size_out; 41 struct mlx5_core_dev *dev = tracer->dev; 42 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 43 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 44 void *mtrc_cap_sp; 45 int err, i; 46 47 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 48 MLX5_REG_MTRC_CAP, 0, 0); 49 if (err) { 50 mlx5_core_warn(dev, "FWTracer: Error reading tracer caps %d\n", 51 err); 52 return err; 53 } 54 55 if (!MLX5_GET(mtrc_cap, out, trace_to_memory)) { 56 mlx5_core_dbg(dev, "FWTracer: Device does not support logging traces to memory\n"); 57 return -ENOTSUPP; 58 } 59 60 tracer->trc_ver = MLX5_GET(mtrc_cap, out, trc_ver); 61 tracer->str_db.first_string_trace = 62 MLX5_GET(mtrc_cap, out, first_string_trace); 63 tracer->str_db.num_string_trace = 64 MLX5_GET(mtrc_cap, out, num_string_trace); 65 tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db); 66 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner); 67 68 for (i = 0; i < tracer->str_db.num_string_db; i++) { 69 mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]); 70 string_db_base_address_out[i] = MLX5_GET(mtrc_string_db_param, 71 mtrc_cap_sp, 72 string_db_base_address); 73 string_db_size_out[i] = MLX5_GET(mtrc_string_db_param, 74 mtrc_cap_sp, string_db_size); 75 } 76 77 return err; 78 } 79 80 static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer *tracer, 81 u32 *out, u32 out_size, 82 u8 trace_owner) 83 { 84 struct mlx5_core_dev *dev = tracer->dev; 85 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 86 87 MLX5_SET(mtrc_cap, in, trace_owner, trace_owner); 88 89 return mlx5_core_access_reg(dev, in, sizeof(in), out, out_size, 90 MLX5_REG_MTRC_CAP, 0, 1); 91 } 92 93 static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer *tracer) 94 { 95 struct mlx5_core_dev *dev = tracer->dev; 96 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 97 int err; 98 99 err = mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out), 100 MLX5_FW_TRACER_ACQUIRE_OWNERSHIP); 101 if (err) { 102 mlx5_core_warn(dev, "FWTracer: Acquire tracer ownership failed %d\n", 103 err); 104 return err; 105 } 106 107 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner); 108 109 if (!tracer->owner) 110 return -EBUSY; 111 112 return 0; 113 } 114 115 static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer) 116 { 117 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 118 119 mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out), 120 MLX5_FW_TRACER_RELEASE_OWNERSHIP); 121 tracer->owner = false; 122 } 123 124 static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer) 125 { 126 struct mlx5_core_dev *dev = tracer->dev; 127 struct device *ddev; 128 dma_addr_t dma; 129 void *buff; 130 gfp_t gfp; 131 int err; 132 133 tracer->buff.size = TRACE_BUFFER_SIZE_BYTE; 134 135 gfp = GFP_KERNEL | __GFP_ZERO; 136 buff = (void *)__get_free_pages(gfp, 137 get_order(tracer->buff.size)); 138 if (!buff) { 139 err = -ENOMEM; 140 mlx5_core_warn(dev, "FWTracer: Failed to allocate pages, %d\n", err); 141 return err; 142 } 143 tracer->buff.log_buf = buff; 144 145 ddev = mlx5_core_dma_dev(dev); 146 dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE); 147 if (dma_mapping_error(ddev, dma)) { 148 mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n", 149 dma_mapping_error(ddev, dma)); 150 err = -ENOMEM; 151 goto free_pages; 152 } 153 tracer->buff.dma = dma; 154 155 return 0; 156 157 free_pages: 158 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size)); 159 160 return err; 161 } 162 163 static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer) 164 { 165 struct mlx5_core_dev *dev = tracer->dev; 166 struct device *ddev; 167 168 if (!tracer->buff.log_buf) 169 return; 170 171 ddev = mlx5_core_dma_dev(dev); 172 dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE); 173 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size)); 174 } 175 176 static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer *tracer) 177 { 178 struct mlx5_core_dev *dev = tracer->dev; 179 int err, inlen, i; 180 __be64 *mtt; 181 void *mkc; 182 u32 *in; 183 184 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + 185 sizeof(*mtt) * round_up(TRACER_BUFFER_PAGE_NUM, 2); 186 187 in = kvzalloc(inlen, GFP_KERNEL); 188 if (!in) 189 return -ENOMEM; 190 191 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 192 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2)); 193 mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 194 for (i = 0 ; i < TRACER_BUFFER_PAGE_NUM ; i++) 195 mtt[i] = cpu_to_be64(tracer->buff.dma + i * PAGE_SIZE); 196 197 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 198 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 199 MLX5_SET(mkc, mkc, lr, 1); 200 MLX5_SET(mkc, mkc, lw, 1); 201 MLX5_SET(mkc, mkc, pd, tracer->buff.pdn); 202 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 203 MLX5_SET(mkc, mkc, qpn, 0xffffff); 204 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 205 MLX5_SET(mkc, mkc, translations_octword_size, 206 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2)); 207 MLX5_SET64(mkc, mkc, start_addr, tracer->buff.dma); 208 MLX5_SET64(mkc, mkc, len, tracer->buff.size); 209 err = mlx5_core_create_mkey(dev, &tracer->buff.mkey, in, inlen); 210 if (err) 211 mlx5_core_warn(dev, "FWTracer: Failed to create mkey, %d\n", err); 212 213 kvfree(in); 214 215 return err; 216 } 217 218 static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer *tracer) 219 { 220 u32 num_string_db = tracer->str_db.num_string_db; 221 int i; 222 223 for (i = 0; i < num_string_db; i++) { 224 kfree(tracer->str_db.buffer[i]); 225 tracer->str_db.buffer[i] = NULL; 226 } 227 } 228 229 static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer) 230 { 231 u32 *string_db_size_out = tracer->str_db.size_out; 232 u32 num_string_db = tracer->str_db.num_string_db; 233 int i; 234 235 for (i = 0; i < num_string_db; i++) { 236 tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL); 237 if (!tracer->str_db.buffer[i]) 238 goto free_strings_db; 239 } 240 241 return 0; 242 243 free_strings_db: 244 mlx5_fw_tracer_free_strings_db(tracer); 245 return -ENOMEM; 246 } 247 248 static void 249 mlx5_fw_tracer_init_saved_traces_array(struct mlx5_fw_tracer *tracer) 250 { 251 tracer->st_arr.saved_traces_index = 0; 252 mutex_init(&tracer->st_arr.lock); 253 } 254 255 static void 256 mlx5_fw_tracer_clean_saved_traces_array(struct mlx5_fw_tracer *tracer) 257 { 258 mutex_destroy(&tracer->st_arr.lock); 259 } 260 261 static void mlx5_tracer_read_strings_db(struct work_struct *work) 262 { 263 struct mlx5_fw_tracer *tracer = container_of(work, struct mlx5_fw_tracer, 264 read_fw_strings_work); 265 u32 num_of_reads, num_string_db = tracer->str_db.num_string_db; 266 struct mlx5_core_dev *dev = tracer->dev; 267 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 268 u32 leftovers, offset; 269 int err = 0, i, j; 270 u32 *out, outlen; 271 void *out_value; 272 273 outlen = MLX5_ST_SZ_BYTES(mtrc_stdb) + STRINGS_DB_READ_SIZE_BYTES; 274 out = kzalloc(outlen, GFP_KERNEL); 275 if (!out) { 276 err = -ENOMEM; 277 goto out; 278 } 279 280 for (i = 0; i < num_string_db; i++) { 281 offset = 0; 282 MLX5_SET(mtrc_stdb, in, string_db_index, i); 283 num_of_reads = tracer->str_db.size_out[i] / 284 STRINGS_DB_READ_SIZE_BYTES; 285 leftovers = (tracer->str_db.size_out[i] % 286 STRINGS_DB_READ_SIZE_BYTES) / 287 STRINGS_DB_LEFTOVER_SIZE_BYTES; 288 289 MLX5_SET(mtrc_stdb, in, read_size, STRINGS_DB_READ_SIZE_BYTES); 290 for (j = 0; j < num_of_reads; j++) { 291 MLX5_SET(mtrc_stdb, in, start_offset, offset); 292 293 err = mlx5_core_access_reg(dev, in, sizeof(in), out, 294 outlen, MLX5_REG_MTRC_STDB, 295 0, 1); 296 if (err) { 297 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n", 298 err); 299 goto out_free; 300 } 301 302 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data); 303 memcpy(tracer->str_db.buffer[i] + offset, out_value, 304 STRINGS_DB_READ_SIZE_BYTES); 305 offset += STRINGS_DB_READ_SIZE_BYTES; 306 } 307 308 /* Strings database is aligned to 64, need to read leftovers*/ 309 MLX5_SET(mtrc_stdb, in, read_size, 310 STRINGS_DB_LEFTOVER_SIZE_BYTES); 311 for (j = 0; j < leftovers; j++) { 312 MLX5_SET(mtrc_stdb, in, start_offset, offset); 313 314 err = mlx5_core_access_reg(dev, in, sizeof(in), out, 315 outlen, MLX5_REG_MTRC_STDB, 316 0, 1); 317 if (err) { 318 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n", 319 err); 320 goto out_free; 321 } 322 323 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data); 324 memcpy(tracer->str_db.buffer[i] + offset, out_value, 325 STRINGS_DB_LEFTOVER_SIZE_BYTES); 326 offset += STRINGS_DB_LEFTOVER_SIZE_BYTES; 327 } 328 } 329 330 tracer->str_db.loaded = true; 331 332 out_free: 333 kfree(out); 334 out: 335 return; 336 } 337 338 static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev) 339 { 340 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 341 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 342 int err; 343 344 MLX5_SET(mtrc_ctrl, in, arm_event, 1); 345 346 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 347 MLX5_REG_MTRC_CTRL, 0, 1); 348 if (err) 349 mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err); 350 } 351 352 static const char *VAL_PARM = "%llx"; 353 static const char *REPLACE_64_VAL_PARM = "%x%x"; 354 static const char *PARAM_CHAR = "%"; 355 356 static int mlx5_tracer_message_hash(u32 message_id) 357 { 358 return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1); 359 } 360 361 static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer, 362 struct tracer_event *tracer_event) 363 { 364 struct hlist_head *head = 365 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)]; 366 struct tracer_string_format *cur_string; 367 368 cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL); 369 if (!cur_string) 370 return NULL; 371 372 hlist_add_head(&cur_string->hlist, head); 373 374 return cur_string; 375 } 376 377 static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer, 378 struct tracer_event *tracer_event) 379 { 380 struct tracer_string_format *cur_string; 381 u32 str_ptr, offset; 382 int i; 383 384 str_ptr = tracer_event->string_event.string_param; 385 386 for (i = 0; i < tracer->str_db.num_string_db; i++) { 387 if (str_ptr > tracer->str_db.base_address_out[i] && 388 str_ptr < tracer->str_db.base_address_out[i] + 389 tracer->str_db.size_out[i]) { 390 offset = str_ptr - tracer->str_db.base_address_out[i]; 391 /* add it to the hash */ 392 cur_string = mlx5_tracer_message_insert(tracer, tracer_event); 393 if (!cur_string) 394 return NULL; 395 cur_string->string = (char *)(tracer->str_db.buffer[i] + 396 offset); 397 return cur_string; 398 } 399 } 400 401 return NULL; 402 } 403 404 static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt) 405 { 406 hlist_del(&str_frmt->hlist); 407 kfree(str_frmt); 408 } 409 410 static int mlx5_tracer_get_num_of_params(char *str) 411 { 412 char *substr, *pstr = str; 413 int num_of_params = 0; 414 415 /* replace %llx with %x%x */ 416 substr = strstr(pstr, VAL_PARM); 417 while (substr) { 418 memcpy(substr, REPLACE_64_VAL_PARM, 4); 419 pstr = substr; 420 substr = strstr(pstr, VAL_PARM); 421 } 422 423 /* count all the % characters */ 424 substr = strstr(str, PARAM_CHAR); 425 while (substr) { 426 num_of_params += 1; 427 str = substr + 1; 428 substr = strstr(str, PARAM_CHAR); 429 } 430 431 return num_of_params; 432 } 433 434 static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head, 435 u8 event_id, u32 tmsn) 436 { 437 struct tracer_string_format *message; 438 439 hlist_for_each_entry(message, head, hlist) 440 if (message->event_id == event_id && message->tmsn == tmsn) 441 return message; 442 443 return NULL; 444 } 445 446 static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer, 447 struct tracer_event *tracer_event) 448 { 449 struct hlist_head *head = 450 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)]; 451 452 return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn); 453 } 454 455 static void poll_trace(struct mlx5_fw_tracer *tracer, 456 struct tracer_event *tracer_event, u64 *trace) 457 { 458 u32 timestamp_low, timestamp_mid, timestamp_high, urts; 459 460 tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id); 461 tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost); 462 463 switch (tracer_event->event_id) { 464 case TRACER_EVENT_TYPE_TIMESTAMP: 465 tracer_event->type = TRACER_EVENT_TYPE_TIMESTAMP; 466 urts = MLX5_GET(tracer_timestamp_event, trace, urts); 467 if (tracer->trc_ver == 0) 468 tracer_event->timestamp_event.unreliable = !!(urts >> 2); 469 else 470 tracer_event->timestamp_event.unreliable = !!(urts & 1); 471 472 timestamp_low = MLX5_GET(tracer_timestamp_event, 473 trace, timestamp7_0); 474 timestamp_mid = MLX5_GET(tracer_timestamp_event, 475 trace, timestamp39_8); 476 timestamp_high = MLX5_GET(tracer_timestamp_event, 477 trace, timestamp52_40); 478 479 tracer_event->timestamp_event.timestamp = 480 ((u64)timestamp_high << 40) | 481 ((u64)timestamp_mid << 8) | 482 (u64)timestamp_low; 483 break; 484 default: 485 if (tracer_event->event_id >= tracer->str_db.first_string_trace || 486 tracer_event->event_id <= tracer->str_db.first_string_trace + 487 tracer->str_db.num_string_trace) { 488 tracer_event->type = TRACER_EVENT_TYPE_STRING; 489 tracer_event->string_event.timestamp = 490 MLX5_GET(tracer_string_event, trace, timestamp); 491 tracer_event->string_event.string_param = 492 MLX5_GET(tracer_string_event, trace, string_param); 493 tracer_event->string_event.tmsn = 494 MLX5_GET(tracer_string_event, trace, tmsn); 495 tracer_event->string_event.tdsn = 496 MLX5_GET(tracer_string_event, trace, tdsn); 497 } else { 498 tracer_event->type = TRACER_EVENT_TYPE_UNRECOGNIZED; 499 } 500 break; 501 } 502 } 503 504 static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event) 505 { 506 struct tracer_event tracer_event; 507 u8 event_id; 508 509 event_id = MLX5_GET(tracer_event, ts_event, event_id); 510 511 if (event_id == TRACER_EVENT_TYPE_TIMESTAMP) 512 poll_trace(tracer, &tracer_event, ts_event); 513 else 514 tracer_event.timestamp_event.timestamp = 0; 515 516 return tracer_event.timestamp_event.timestamp; 517 } 518 519 static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer) 520 { 521 struct tracer_string_format *str_frmt; 522 struct hlist_node *n; 523 int i; 524 525 for (i = 0; i < MESSAGE_HASH_SIZE; i++) { 526 hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist) 527 mlx5_tracer_clean_message(str_frmt); 528 } 529 } 530 531 static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer) 532 { 533 struct tracer_string_format *str_frmt, *tmp_str; 534 535 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, 536 list) 537 list_del(&str_frmt->list); 538 } 539 540 static void mlx5_fw_tracer_save_trace(struct mlx5_fw_tracer *tracer, 541 u64 timestamp, bool lost, 542 u8 event_id, char *msg) 543 { 544 struct mlx5_fw_trace_data *trace_data; 545 546 mutex_lock(&tracer->st_arr.lock); 547 trace_data = &tracer->st_arr.straces[tracer->st_arr.saved_traces_index]; 548 trace_data->timestamp = timestamp; 549 trace_data->lost = lost; 550 trace_data->event_id = event_id; 551 strscpy_pad(trace_data->msg, msg, TRACE_STR_MSG); 552 553 tracer->st_arr.saved_traces_index = 554 (tracer->st_arr.saved_traces_index + 1) & (SAVED_TRACES_NUM - 1); 555 mutex_unlock(&tracer->st_arr.lock); 556 } 557 558 static noinline 559 void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt, 560 struct mlx5_core_dev *dev, 561 u64 trace_timestamp) 562 { 563 char tmp[512]; 564 565 snprintf(tmp, sizeof(tmp), str_frmt->string, 566 str_frmt->params[0], 567 str_frmt->params[1], 568 str_frmt->params[2], 569 str_frmt->params[3], 570 str_frmt->params[4], 571 str_frmt->params[5], 572 str_frmt->params[6]); 573 574 trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost, 575 str_frmt->event_id, tmp); 576 577 mlx5_fw_tracer_save_trace(dev->tracer, trace_timestamp, 578 str_frmt->lost, str_frmt->event_id, tmp); 579 580 /* remove it from hash */ 581 mlx5_tracer_clean_message(str_frmt); 582 } 583 584 static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, 585 struct tracer_event *tracer_event) 586 { 587 struct tracer_string_format *cur_string; 588 589 if (tracer_event->string_event.tdsn == 0) { 590 cur_string = mlx5_tracer_get_string(tracer, tracer_event); 591 if (!cur_string) 592 return -1; 593 594 cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string); 595 cur_string->last_param_num = 0; 596 cur_string->event_id = tracer_event->event_id; 597 cur_string->tmsn = tracer_event->string_event.tmsn; 598 cur_string->timestamp = tracer_event->string_event.timestamp; 599 cur_string->lost = tracer_event->lost_event; 600 if (cur_string->num_of_params == 0) /* trace with no params */ 601 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 602 } else { 603 cur_string = mlx5_tracer_message_get(tracer, tracer_event); 604 if (!cur_string) { 605 pr_debug("%s Got string event for unknown string tdsm: %d\n", 606 __func__, tracer_event->string_event.tmsn); 607 return -1; 608 } 609 cur_string->last_param_num += 1; 610 if (cur_string->last_param_num > TRACER_MAX_PARAMS) { 611 pr_debug("%s Number of params exceeds the max (%d)\n", 612 __func__, TRACER_MAX_PARAMS); 613 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 614 return 0; 615 } 616 /* keep the new parameter */ 617 cur_string->params[cur_string->last_param_num - 1] = 618 tracer_event->string_event.string_param; 619 if (cur_string->last_param_num == cur_string->num_of_params) 620 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 621 } 622 623 return 0; 624 } 625 626 static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer, 627 struct tracer_event *tracer_event) 628 { 629 struct tracer_timestamp_event timestamp_event = 630 tracer_event->timestamp_event; 631 struct tracer_string_format *str_frmt, *tmp_str; 632 struct mlx5_core_dev *dev = tracer->dev; 633 u64 trace_timestamp; 634 635 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) { 636 list_del(&str_frmt->list); 637 if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0)) 638 trace_timestamp = (timestamp_event.timestamp & MASK_52_7) | 639 (str_frmt->timestamp & MASK_6_0); 640 else 641 trace_timestamp = ((timestamp_event.timestamp - 1) & MASK_52_7) | 642 (str_frmt->timestamp & MASK_6_0); 643 644 mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp); 645 } 646 } 647 648 static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer, 649 struct tracer_event *tracer_event) 650 { 651 if (tracer_event->type == TRACER_EVENT_TYPE_STRING) { 652 mlx5_tracer_handle_string_trace(tracer, tracer_event); 653 } else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) { 654 if (!tracer_event->timestamp_event.unreliable) 655 mlx5_tracer_handle_timestamp_trace(tracer, tracer_event); 656 } else { 657 pr_debug("%s Got unrecognised type %d for parsing, exiting..\n", 658 __func__, tracer_event->type); 659 } 660 return 0; 661 } 662 663 static void mlx5_fw_tracer_handle_traces(struct work_struct *work) 664 { 665 struct mlx5_fw_tracer *tracer = 666 container_of(work, struct mlx5_fw_tracer, handle_traces_work); 667 u64 block_timestamp, last_block_timestamp, tmp_trace_block[TRACES_PER_BLOCK]; 668 u32 block_count, start_offset, prev_start_offset, prev_consumer_index; 669 u32 trace_event_size = MLX5_ST_SZ_BYTES(tracer_event); 670 struct mlx5_core_dev *dev = tracer->dev; 671 struct tracer_event tracer_event; 672 int i; 673 674 mlx5_core_dbg(dev, "FWTracer: Handle Trace event, owner=(%d)\n", tracer->owner); 675 if (!tracer->owner) 676 return; 677 678 if (unlikely(!tracer->str_db.loaded)) 679 goto arm; 680 681 block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE; 682 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE; 683 684 /* Copy the block to local buffer to avoid HW override while being processed */ 685 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset, 686 TRACER_BLOCK_SIZE_BYTE); 687 688 block_timestamp = 689 get_block_timestamp(tracer, &tmp_trace_block[TRACES_PER_BLOCK - 1]); 690 691 while (block_timestamp > tracer->last_timestamp) { 692 /* Check block override if it's not the first block */ 693 if (!tracer->last_timestamp) { 694 u64 *ts_event; 695 /* To avoid block override be the HW in case of buffer 696 * wraparound, the time stamp of the previous block 697 * should be compared to the last timestamp handled 698 * by the driver. 699 */ 700 prev_consumer_index = 701 (tracer->buff.consumer_index - 1) & (block_count - 1); 702 prev_start_offset = prev_consumer_index * TRACER_BLOCK_SIZE_BYTE; 703 704 ts_event = tracer->buff.log_buf + prev_start_offset + 705 (TRACES_PER_BLOCK - 1) * trace_event_size; 706 last_block_timestamp = get_block_timestamp(tracer, ts_event); 707 /* If previous timestamp different from last stored 708 * timestamp then there is a good chance that the 709 * current buffer is overwritten and therefore should 710 * not be parsed. 711 */ 712 if (tracer->last_timestamp != last_block_timestamp) { 713 mlx5_core_warn(dev, "FWTracer: Events were lost\n"); 714 tracer->last_timestamp = block_timestamp; 715 tracer->buff.consumer_index = 716 (tracer->buff.consumer_index + 1) & (block_count - 1); 717 break; 718 } 719 } 720 721 /* Parse events */ 722 for (i = 0; i < TRACES_PER_BLOCK ; i++) { 723 poll_trace(tracer, &tracer_event, &tmp_trace_block[i]); 724 mlx5_tracer_handle_trace(tracer, &tracer_event); 725 } 726 727 tracer->buff.consumer_index = 728 (tracer->buff.consumer_index + 1) & (block_count - 1); 729 730 tracer->last_timestamp = block_timestamp; 731 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE; 732 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset, 733 TRACER_BLOCK_SIZE_BYTE); 734 block_timestamp = get_block_timestamp(tracer, 735 &tmp_trace_block[TRACES_PER_BLOCK - 1]); 736 } 737 738 arm: 739 mlx5_fw_tracer_arm(dev); 740 } 741 742 static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer) 743 { 744 struct mlx5_core_dev *dev = tracer->dev; 745 u32 out[MLX5_ST_SZ_DW(mtrc_conf)] = {0}; 746 u32 in[MLX5_ST_SZ_DW(mtrc_conf)] = {0}; 747 int err; 748 749 MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY); 750 MLX5_SET(mtrc_conf, in, log_trace_buffer_size, 751 ilog2(TRACER_BUFFER_PAGE_NUM)); 752 MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey); 753 754 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 755 MLX5_REG_MTRC_CONF, 0, 1); 756 if (err) 757 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err); 758 759 return err; 760 } 761 762 static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer *tracer, u8 status, u8 arm) 763 { 764 struct mlx5_core_dev *dev = tracer->dev; 765 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 766 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 767 int err; 768 769 MLX5_SET(mtrc_ctrl, in, modify_field_select, TRACE_STATUS); 770 MLX5_SET(mtrc_ctrl, in, trace_status, status); 771 MLX5_SET(mtrc_ctrl, in, arm_event, arm); 772 773 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 774 MLX5_REG_MTRC_CTRL, 0, 1); 775 776 if (!err && status) 777 tracer->last_timestamp = 0; 778 779 return err; 780 } 781 782 static int mlx5_fw_tracer_start(struct mlx5_fw_tracer *tracer) 783 { 784 struct mlx5_core_dev *dev = tracer->dev; 785 int err; 786 787 err = mlx5_fw_tracer_ownership_acquire(tracer); 788 if (err) { 789 mlx5_core_dbg(dev, "FWTracer: Ownership was not granted %d\n", err); 790 /* Don't fail since ownership can be acquired on a later FW event */ 791 return 0; 792 } 793 794 err = mlx5_fw_tracer_set_mtrc_conf(tracer); 795 if (err) { 796 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configuration %d\n", err); 797 goto release_ownership; 798 } 799 800 /* enable tracer & trace events */ 801 err = mlx5_fw_tracer_set_mtrc_ctrl(tracer, 1, 1); 802 if (err) { 803 mlx5_core_warn(dev, "FWTracer: Failed to enable tracer %d\n", err); 804 goto release_ownership; 805 } 806 807 mlx5_core_dbg(dev, "FWTracer: Ownership granted and active\n"); 808 return 0; 809 810 release_ownership: 811 mlx5_fw_tracer_ownership_release(tracer); 812 return err; 813 } 814 815 static void mlx5_fw_tracer_ownership_change(struct work_struct *work) 816 { 817 struct mlx5_fw_tracer *tracer = 818 container_of(work, struct mlx5_fw_tracer, ownership_change_work); 819 820 mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner); 821 if (tracer->owner) { 822 tracer->owner = false; 823 tracer->buff.consumer_index = 0; 824 return; 825 } 826 827 mlx5_fw_tracer_start(tracer); 828 } 829 830 static int mlx5_fw_tracer_set_core_dump_reg(struct mlx5_core_dev *dev, 831 u32 *in, int size_in) 832 { 833 u32 out[MLX5_ST_SZ_DW(core_dump_reg)] = {}; 834 835 if (!MLX5_CAP_DEBUG(dev, core_dump_general) && 836 !MLX5_CAP_DEBUG(dev, core_dump_qp)) 837 return -EOPNOTSUPP; 838 839 return mlx5_core_access_reg(dev, in, size_in, out, sizeof(out), 840 MLX5_REG_CORE_DUMP, 0, 1); 841 } 842 843 int mlx5_fw_tracer_trigger_core_dump_general(struct mlx5_core_dev *dev) 844 { 845 struct mlx5_fw_tracer *tracer = dev->tracer; 846 u32 in[MLX5_ST_SZ_DW(core_dump_reg)] = {}; 847 int err; 848 849 if (!MLX5_CAP_DEBUG(dev, core_dump_general) || !tracer) 850 return -EOPNOTSUPP; 851 if (!tracer->owner) 852 return -EPERM; 853 854 MLX5_SET(core_dump_reg, in, core_dump_type, 0x0); 855 856 err = mlx5_fw_tracer_set_core_dump_reg(dev, in, sizeof(in)); 857 if (err) 858 return err; 859 queue_work(tracer->work_queue, &tracer->handle_traces_work); 860 flush_workqueue(tracer->work_queue); 861 return 0; 862 } 863 864 static int 865 mlx5_devlink_fmsg_fill_trace(struct devlink_fmsg *fmsg, 866 struct mlx5_fw_trace_data *trace_data) 867 { 868 int err; 869 870 err = devlink_fmsg_obj_nest_start(fmsg); 871 if (err) 872 return err; 873 874 err = devlink_fmsg_u64_pair_put(fmsg, "timestamp", trace_data->timestamp); 875 if (err) 876 return err; 877 878 err = devlink_fmsg_bool_pair_put(fmsg, "lost", trace_data->lost); 879 if (err) 880 return err; 881 882 err = devlink_fmsg_u8_pair_put(fmsg, "event_id", trace_data->event_id); 883 if (err) 884 return err; 885 886 err = devlink_fmsg_string_pair_put(fmsg, "msg", trace_data->msg); 887 if (err) 888 return err; 889 890 err = devlink_fmsg_obj_nest_end(fmsg); 891 if (err) 892 return err; 893 return 0; 894 } 895 896 int mlx5_fw_tracer_get_saved_traces_objects(struct mlx5_fw_tracer *tracer, 897 struct devlink_fmsg *fmsg) 898 { 899 struct mlx5_fw_trace_data *straces = tracer->st_arr.straces; 900 u32 index, start_index, end_index; 901 u32 saved_traces_index; 902 int err; 903 904 if (!straces[0].timestamp) 905 return -ENOMSG; 906 907 mutex_lock(&tracer->st_arr.lock); 908 saved_traces_index = tracer->st_arr.saved_traces_index; 909 if (straces[saved_traces_index].timestamp) 910 start_index = saved_traces_index; 911 else 912 start_index = 0; 913 end_index = (saved_traces_index - 1) & (SAVED_TRACES_NUM - 1); 914 915 err = devlink_fmsg_arr_pair_nest_start(fmsg, "dump fw traces"); 916 if (err) 917 goto unlock; 918 index = start_index; 919 while (index != end_index) { 920 err = mlx5_devlink_fmsg_fill_trace(fmsg, &straces[index]); 921 if (err) 922 goto unlock; 923 924 index = (index + 1) & (SAVED_TRACES_NUM - 1); 925 } 926 927 err = devlink_fmsg_arr_pair_nest_end(fmsg); 928 unlock: 929 mutex_unlock(&tracer->st_arr.lock); 930 return err; 931 } 932 933 /* Create software resources (Buffers, etc ..) */ 934 struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev) 935 { 936 struct mlx5_fw_tracer *tracer = NULL; 937 int err; 938 939 if (!MLX5_CAP_MCAM_REG(dev, tracer_registers)) { 940 mlx5_core_dbg(dev, "FWTracer: Tracer capability not present\n"); 941 return NULL; 942 } 943 944 tracer = kvzalloc(sizeof(*tracer), GFP_KERNEL); 945 if (!tracer) 946 return ERR_PTR(-ENOMEM); 947 948 tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer"); 949 if (!tracer->work_queue) { 950 err = -ENOMEM; 951 goto free_tracer; 952 } 953 954 tracer->dev = dev; 955 956 INIT_LIST_HEAD(&tracer->ready_strings_list); 957 INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change); 958 INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db); 959 INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces); 960 961 962 err = mlx5_query_mtrc_caps(tracer); 963 if (err) { 964 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err); 965 goto destroy_workqueue; 966 } 967 968 err = mlx5_fw_tracer_create_log_buf(tracer); 969 if (err) { 970 mlx5_core_warn(dev, "FWTracer: Create log buffer failed %d\n", err); 971 goto destroy_workqueue; 972 } 973 974 err = mlx5_fw_tracer_allocate_strings_db(tracer); 975 if (err) { 976 mlx5_core_warn(dev, "FWTracer: Allocate strings database failed %d\n", err); 977 goto free_log_buf; 978 } 979 980 mlx5_fw_tracer_init_saved_traces_array(tracer); 981 mlx5_core_dbg(dev, "FWTracer: Tracer created\n"); 982 983 return tracer; 984 985 free_log_buf: 986 mlx5_fw_tracer_destroy_log_buf(tracer); 987 destroy_workqueue: 988 tracer->dev = NULL; 989 destroy_workqueue(tracer->work_queue); 990 free_tracer: 991 kvfree(tracer); 992 return ERR_PTR(err); 993 } 994 995 static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data); 996 997 /* Create HW resources + start tracer */ 998 int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer) 999 { 1000 struct mlx5_core_dev *dev; 1001 int err; 1002 1003 if (IS_ERR_OR_NULL(tracer)) 1004 return 0; 1005 1006 dev = tracer->dev; 1007 1008 if (!tracer->str_db.loaded) 1009 queue_work(tracer->work_queue, &tracer->read_fw_strings_work); 1010 1011 err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn); 1012 if (err) { 1013 mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err); 1014 goto err_cancel_work; 1015 } 1016 1017 err = mlx5_fw_tracer_create_mkey(tracer); 1018 if (err) { 1019 mlx5_core_warn(dev, "FWTracer: Failed to create mkey %d\n", err); 1020 goto err_dealloc_pd; 1021 } 1022 1023 MLX5_NB_INIT(&tracer->nb, fw_tracer_event, DEVICE_TRACER); 1024 mlx5_eq_notifier_register(dev, &tracer->nb); 1025 1026 err = mlx5_fw_tracer_start(tracer); 1027 if (err) { 1028 mlx5_core_warn(dev, "FWTracer: Failed to start tracer %d\n", err); 1029 goto err_notifier_unregister; 1030 } 1031 return 0; 1032 1033 err_notifier_unregister: 1034 mlx5_eq_notifier_unregister(dev, &tracer->nb); 1035 mlx5_core_destroy_mkey(dev, tracer->buff.mkey); 1036 err_dealloc_pd: 1037 mlx5_core_dealloc_pd(dev, tracer->buff.pdn); 1038 err_cancel_work: 1039 cancel_work_sync(&tracer->read_fw_strings_work); 1040 return err; 1041 } 1042 1043 /* Stop tracer + Cleanup HW resources */ 1044 void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer) 1045 { 1046 if (IS_ERR_OR_NULL(tracer)) 1047 return; 1048 1049 mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n", 1050 tracer->owner); 1051 mlx5_eq_notifier_unregister(tracer->dev, &tracer->nb); 1052 cancel_work_sync(&tracer->ownership_change_work); 1053 cancel_work_sync(&tracer->handle_traces_work); 1054 1055 if (tracer->owner) 1056 mlx5_fw_tracer_ownership_release(tracer); 1057 1058 mlx5_core_destroy_mkey(tracer->dev, tracer->buff.mkey); 1059 mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn); 1060 } 1061 1062 /* Free software resources (Buffers, etc ..) */ 1063 void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer) 1064 { 1065 if (IS_ERR_OR_NULL(tracer)) 1066 return; 1067 1068 mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n"); 1069 1070 cancel_work_sync(&tracer->read_fw_strings_work); 1071 mlx5_fw_tracer_clean_ready_list(tracer); 1072 mlx5_fw_tracer_clean_print_hash(tracer); 1073 mlx5_fw_tracer_clean_saved_traces_array(tracer); 1074 mlx5_fw_tracer_free_strings_db(tracer); 1075 mlx5_fw_tracer_destroy_log_buf(tracer); 1076 destroy_workqueue(tracer->work_queue); 1077 kvfree(tracer); 1078 } 1079 1080 static int mlx5_fw_tracer_recreate_strings_db(struct mlx5_fw_tracer *tracer) 1081 { 1082 struct mlx5_core_dev *dev; 1083 int err; 1084 1085 cancel_work_sync(&tracer->read_fw_strings_work); 1086 mlx5_fw_tracer_clean_ready_list(tracer); 1087 mlx5_fw_tracer_clean_print_hash(tracer); 1088 mlx5_fw_tracer_clean_saved_traces_array(tracer); 1089 mlx5_fw_tracer_free_strings_db(tracer); 1090 1091 dev = tracer->dev; 1092 err = mlx5_query_mtrc_caps(tracer); 1093 if (err) { 1094 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err); 1095 return err; 1096 } 1097 1098 err = mlx5_fw_tracer_allocate_strings_db(tracer); 1099 if (err) { 1100 mlx5_core_warn(dev, "FWTracer: Allocate strings DB failed %d\n", err); 1101 return err; 1102 } 1103 mlx5_fw_tracer_init_saved_traces_array(tracer); 1104 1105 return 0; 1106 } 1107 1108 int mlx5_fw_tracer_reload(struct mlx5_fw_tracer *tracer) 1109 { 1110 struct mlx5_core_dev *dev; 1111 int err; 1112 1113 if (IS_ERR_OR_NULL(tracer)) 1114 return 0; 1115 1116 dev = tracer->dev; 1117 mlx5_fw_tracer_cleanup(tracer); 1118 err = mlx5_fw_tracer_recreate_strings_db(tracer); 1119 if (err) { 1120 mlx5_core_warn(dev, "Failed to recreate FW tracer strings DB\n"); 1121 return err; 1122 } 1123 err = mlx5_fw_tracer_init(tracer); 1124 if (err) { 1125 mlx5_core_warn(dev, "Failed to re-initialize FW tracer\n"); 1126 return err; 1127 } 1128 1129 return 0; 1130 } 1131 1132 static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data) 1133 { 1134 struct mlx5_fw_tracer *tracer = mlx5_nb_cof(nb, struct mlx5_fw_tracer, nb); 1135 struct mlx5_core_dev *dev = tracer->dev; 1136 struct mlx5_eqe *eqe = data; 1137 1138 switch (eqe->sub_type) { 1139 case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE: 1140 queue_work(tracer->work_queue, &tracer->ownership_change_work); 1141 break; 1142 case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE: 1143 queue_work(tracer->work_queue, &tracer->handle_traces_work); 1144 break; 1145 default: 1146 mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n", 1147 eqe->sub_type); 1148 } 1149 1150 return NOTIFY_OK; 1151 } 1152 1153 EXPORT_TRACEPOINT_SYMBOL(mlx5_fw); 1154