1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(C) 2015-2018 Linaro Limited. 4 * 5 * Author: Tor Jeremiassen <tor@ti.com> 6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/err.h> 11 #include <linux/kernel.h> 12 #include <linux/log2.h> 13 #include <linux/types.h> 14 #include <linux/zalloc.h> 15 16 #include <opencsd/ocsd_if_types.h> 17 #include <stdlib.h> 18 19 #include "auxtrace.h" 20 #include "color.h" 21 #include "cs-etm.h" 22 #include "cs-etm-decoder/cs-etm-decoder.h" 23 #include "debug.h" 24 #include "dso.h" 25 #include "evlist.h" 26 #include "intlist.h" 27 #include "machine.h" 28 #include "map.h" 29 #include "perf.h" 30 #include "session.h" 31 #include "map_symbol.h" 32 #include "branch.h" 33 #include "symbol.h" 34 #include "tool.h" 35 #include "thread.h" 36 #include "thread-stack.h" 37 #include <tools/libc_compat.h> 38 #include "util/synthetic-events.h" 39 40 #define MAX_TIMESTAMP (~0ULL) 41 42 struct cs_etm_auxtrace { 43 struct auxtrace auxtrace; 44 struct auxtrace_queues queues; 45 struct auxtrace_heap heap; 46 struct itrace_synth_opts synth_opts; 47 struct perf_session *session; 48 struct machine *machine; 49 struct thread *unknown_thread; 50 51 u8 timeless_decoding; 52 u8 snapshot_mode; 53 u8 data_queued; 54 u8 sample_branches; 55 u8 sample_instructions; 56 57 int num_cpu; 58 u32 auxtrace_type; 59 u64 branches_sample_type; 60 u64 branches_id; 61 u64 instructions_sample_type; 62 u64 instructions_sample_period; 63 u64 instructions_id; 64 u64 **metadata; 65 u64 kernel_start; 66 unsigned int pmu_type; 67 }; 68 69 struct cs_etm_traceid_queue { 70 u8 trace_chan_id; 71 pid_t pid, tid; 72 u64 period_instructions; 73 size_t last_branch_pos; 74 union perf_event *event_buf; 75 struct thread *thread; 76 struct branch_stack *last_branch; 77 struct branch_stack *last_branch_rb; 78 struct cs_etm_packet *prev_packet; 79 struct cs_etm_packet *packet; 80 struct cs_etm_packet_queue packet_queue; 81 }; 82 83 struct cs_etm_queue { 84 struct cs_etm_auxtrace *etm; 85 struct cs_etm_decoder *decoder; 86 struct auxtrace_buffer *buffer; 87 unsigned int queue_nr; 88 u8 pending_timestamp; 89 u64 offset; 90 const unsigned char *buf; 91 size_t buf_len, buf_used; 92 /* Conversion between traceID and index in traceid_queues array */ 93 struct intlist *traceid_queues_list; 94 struct cs_etm_traceid_queue **traceid_queues; 95 }; 96 97 /* RB tree for quick conversion between traceID and metadata pointers */ 98 static struct intlist *traceid_list; 99 100 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm); 101 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm); 102 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, 103 pid_t tid); 104 static int cs_etm__get_data_block(struct cs_etm_queue *etmq); 105 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq); 106 107 /* PTMs ETMIDR [11:8] set to b0011 */ 108 #define ETMIDR_PTM_VERSION 0x00000300 109 110 /* 111 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to 112 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply 113 * encode the etm queue number as the upper 16 bit and the channel as 114 * the lower 16 bit. 115 */ 116 #define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \ 117 (queue_nr << 16 | trace_chan_id) 118 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16) 119 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff) 120 121 static u32 cs_etm__get_v7_protocol_version(u32 etmidr) 122 { 123 etmidr &= ETMIDR_PTM_VERSION; 124 125 if (etmidr == ETMIDR_PTM_VERSION) 126 return CS_ETM_PROTO_PTM; 127 128 return CS_ETM_PROTO_ETMV3; 129 } 130 131 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic) 132 { 133 struct int_node *inode; 134 u64 *metadata; 135 136 inode = intlist__find(traceid_list, trace_chan_id); 137 if (!inode) 138 return -EINVAL; 139 140 metadata = inode->priv; 141 *magic = metadata[CS_ETM_MAGIC]; 142 return 0; 143 } 144 145 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu) 146 { 147 struct int_node *inode; 148 u64 *metadata; 149 150 inode = intlist__find(traceid_list, trace_chan_id); 151 if (!inode) 152 return -EINVAL; 153 154 metadata = inode->priv; 155 *cpu = (int)metadata[CS_ETM_CPU]; 156 return 0; 157 } 158 159 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, 160 u8 trace_chan_id) 161 { 162 /* 163 * Wnen a timestamp packet is encountered the backend code 164 * is stopped so that the front end has time to process packets 165 * that were accumulated in the traceID queue. Since there can 166 * be more than one channel per cs_etm_queue, we need to specify 167 * what traceID queue needs servicing. 168 */ 169 etmq->pending_timestamp = trace_chan_id; 170 } 171 172 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq, 173 u8 *trace_chan_id) 174 { 175 struct cs_etm_packet_queue *packet_queue; 176 177 if (!etmq->pending_timestamp) 178 return 0; 179 180 if (trace_chan_id) 181 *trace_chan_id = etmq->pending_timestamp; 182 183 packet_queue = cs_etm__etmq_get_packet_queue(etmq, 184 etmq->pending_timestamp); 185 if (!packet_queue) 186 return 0; 187 188 /* Acknowledge pending status */ 189 etmq->pending_timestamp = 0; 190 191 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */ 192 return packet_queue->timestamp; 193 } 194 195 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue) 196 { 197 int i; 198 199 queue->head = 0; 200 queue->tail = 0; 201 queue->packet_count = 0; 202 for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) { 203 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN; 204 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR; 205 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR; 206 queue->packet_buffer[i].instr_count = 0; 207 queue->packet_buffer[i].last_instr_taken_branch = false; 208 queue->packet_buffer[i].last_instr_size = 0; 209 queue->packet_buffer[i].last_instr_type = 0; 210 queue->packet_buffer[i].last_instr_subtype = 0; 211 queue->packet_buffer[i].last_instr_cond = 0; 212 queue->packet_buffer[i].flags = 0; 213 queue->packet_buffer[i].exception_number = UINT32_MAX; 214 queue->packet_buffer[i].trace_chan_id = UINT8_MAX; 215 queue->packet_buffer[i].cpu = INT_MIN; 216 } 217 } 218 219 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq) 220 { 221 int idx; 222 struct int_node *inode; 223 struct cs_etm_traceid_queue *tidq; 224 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 225 226 intlist__for_each_entry(inode, traceid_queues_list) { 227 idx = (int)(intptr_t)inode->priv; 228 tidq = etmq->traceid_queues[idx]; 229 cs_etm__clear_packet_queue(&tidq->packet_queue); 230 } 231 } 232 233 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, 234 struct cs_etm_traceid_queue *tidq, 235 u8 trace_chan_id) 236 { 237 int rc = -ENOMEM; 238 struct auxtrace_queue *queue; 239 struct cs_etm_auxtrace *etm = etmq->etm; 240 241 cs_etm__clear_packet_queue(&tidq->packet_queue); 242 243 queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; 244 tidq->tid = queue->tid; 245 tidq->pid = -1; 246 tidq->trace_chan_id = trace_chan_id; 247 248 tidq->packet = zalloc(sizeof(struct cs_etm_packet)); 249 if (!tidq->packet) 250 goto out; 251 252 tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet)); 253 if (!tidq->prev_packet) 254 goto out_free; 255 256 if (etm->synth_opts.last_branch) { 257 size_t sz = sizeof(struct branch_stack); 258 259 sz += etm->synth_opts.last_branch_sz * 260 sizeof(struct branch_entry); 261 tidq->last_branch = zalloc(sz); 262 if (!tidq->last_branch) 263 goto out_free; 264 tidq->last_branch_rb = zalloc(sz); 265 if (!tidq->last_branch_rb) 266 goto out_free; 267 } 268 269 tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); 270 if (!tidq->event_buf) 271 goto out_free; 272 273 return 0; 274 275 out_free: 276 zfree(&tidq->last_branch_rb); 277 zfree(&tidq->last_branch); 278 zfree(&tidq->prev_packet); 279 zfree(&tidq->packet); 280 out: 281 return rc; 282 } 283 284 static struct cs_etm_traceid_queue 285 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) 286 { 287 int idx; 288 struct int_node *inode; 289 struct intlist *traceid_queues_list; 290 struct cs_etm_traceid_queue *tidq, **traceid_queues; 291 struct cs_etm_auxtrace *etm = etmq->etm; 292 293 if (etm->timeless_decoding) 294 trace_chan_id = CS_ETM_PER_THREAD_TRACEID; 295 296 traceid_queues_list = etmq->traceid_queues_list; 297 298 /* 299 * Check if the traceid_queue exist for this traceID by looking 300 * in the queue list. 301 */ 302 inode = intlist__find(traceid_queues_list, trace_chan_id); 303 if (inode) { 304 idx = (int)(intptr_t)inode->priv; 305 return etmq->traceid_queues[idx]; 306 } 307 308 /* We couldn't find a traceid_queue for this traceID, allocate one */ 309 tidq = malloc(sizeof(*tidq)); 310 if (!tidq) 311 return NULL; 312 313 memset(tidq, 0, sizeof(*tidq)); 314 315 /* Get a valid index for the new traceid_queue */ 316 idx = intlist__nr_entries(traceid_queues_list); 317 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */ 318 inode = intlist__findnew(traceid_queues_list, trace_chan_id); 319 if (!inode) 320 goto out_free; 321 322 /* Associate this traceID with this index */ 323 inode->priv = (void *)(intptr_t)idx; 324 325 if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id)) 326 goto out_free; 327 328 /* Grow the traceid_queues array by one unit */ 329 traceid_queues = etmq->traceid_queues; 330 traceid_queues = reallocarray(traceid_queues, 331 idx + 1, 332 sizeof(*traceid_queues)); 333 334 /* 335 * On failure reallocarray() returns NULL and the original block of 336 * memory is left untouched. 337 */ 338 if (!traceid_queues) 339 goto out_free; 340 341 traceid_queues[idx] = tidq; 342 etmq->traceid_queues = traceid_queues; 343 344 return etmq->traceid_queues[idx]; 345 346 out_free: 347 /* 348 * Function intlist__remove() removes the inode from the list 349 * and delete the memory associated to it. 350 */ 351 intlist__remove(traceid_queues_list, inode); 352 free(tidq); 353 354 return NULL; 355 } 356 357 struct cs_etm_packet_queue 358 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) 359 { 360 struct cs_etm_traceid_queue *tidq; 361 362 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 363 if (tidq) 364 return &tidq->packet_queue; 365 366 return NULL; 367 } 368 369 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, 370 struct cs_etm_traceid_queue *tidq) 371 { 372 struct cs_etm_packet *tmp; 373 374 if (etm->sample_branches || etm->synth_opts.last_branch || 375 etm->sample_instructions) { 376 /* 377 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for 378 * the next incoming packet. 379 */ 380 tmp = tidq->packet; 381 tidq->packet = tidq->prev_packet; 382 tidq->prev_packet = tmp; 383 } 384 } 385 386 static void cs_etm__packet_dump(const char *pkt_string) 387 { 388 const char *color = PERF_COLOR_BLUE; 389 int len = strlen(pkt_string); 390 391 if (len && (pkt_string[len-1] == '\n')) 392 color_fprintf(stdout, color, " %s", pkt_string); 393 else 394 color_fprintf(stdout, color, " %s\n", pkt_string); 395 396 fflush(stdout); 397 } 398 399 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params, 400 struct cs_etm_auxtrace *etm, int idx, 401 u32 etmidr) 402 { 403 u64 **metadata = etm->metadata; 404 405 t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr); 406 t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR]; 407 t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR]; 408 } 409 410 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params, 411 struct cs_etm_auxtrace *etm, int idx) 412 { 413 u64 **metadata = etm->metadata; 414 415 t_params[idx].protocol = CS_ETM_PROTO_ETMV4i; 416 t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0]; 417 t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1]; 418 t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2]; 419 t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8]; 420 t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR]; 421 t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR]; 422 } 423 424 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params, 425 struct cs_etm_auxtrace *etm) 426 { 427 int i; 428 u32 etmidr; 429 u64 architecture; 430 431 for (i = 0; i < etm->num_cpu; i++) { 432 architecture = etm->metadata[i][CS_ETM_MAGIC]; 433 434 switch (architecture) { 435 case __perf_cs_etmv3_magic: 436 etmidr = etm->metadata[i][CS_ETM_ETMIDR]; 437 cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr); 438 break; 439 case __perf_cs_etmv4_magic: 440 cs_etm__set_trace_param_etmv4(t_params, etm, i); 441 break; 442 default: 443 return -EINVAL; 444 } 445 } 446 447 return 0; 448 } 449 450 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params, 451 struct cs_etm_queue *etmq, 452 enum cs_etm_decoder_operation mode) 453 { 454 int ret = -EINVAL; 455 456 if (!(mode < CS_ETM_OPERATION_MAX)) 457 goto out; 458 459 d_params->packet_printer = cs_etm__packet_dump; 460 d_params->operation = mode; 461 d_params->data = etmq; 462 d_params->formatted = true; 463 d_params->fsyncs = false; 464 d_params->hsyncs = false; 465 d_params->frame_aligned = true; 466 467 ret = 0; 468 out: 469 return ret; 470 } 471 472 static void cs_etm__dump_event(struct cs_etm_auxtrace *etm, 473 struct auxtrace_buffer *buffer) 474 { 475 int ret; 476 const char *color = PERF_COLOR_BLUE; 477 struct cs_etm_decoder_params d_params; 478 struct cs_etm_trace_params *t_params; 479 struct cs_etm_decoder *decoder; 480 size_t buffer_used = 0; 481 482 fprintf(stdout, "\n"); 483 color_fprintf(stdout, color, 484 ". ... CoreSight ETM Trace data: size %zu bytes\n", 485 buffer->size); 486 487 /* Use metadata to fill in trace parameters for trace decoder */ 488 t_params = zalloc(sizeof(*t_params) * etm->num_cpu); 489 490 if (!t_params) 491 return; 492 493 if (cs_etm__init_trace_params(t_params, etm)) 494 goto out_free; 495 496 /* Set decoder parameters to simply print the trace packets */ 497 if (cs_etm__init_decoder_params(&d_params, NULL, 498 CS_ETM_OPERATION_PRINT)) 499 goto out_free; 500 501 decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params); 502 503 if (!decoder) 504 goto out_free; 505 do { 506 size_t consumed; 507 508 ret = cs_etm_decoder__process_data_block( 509 decoder, buffer->offset, 510 &((u8 *)buffer->data)[buffer_used], 511 buffer->size - buffer_used, &consumed); 512 if (ret) 513 break; 514 515 buffer_used += consumed; 516 } while (buffer_used < buffer->size); 517 518 cs_etm_decoder__free(decoder); 519 520 out_free: 521 zfree(&t_params); 522 } 523 524 static int cs_etm__flush_events(struct perf_session *session, 525 struct perf_tool *tool) 526 { 527 int ret; 528 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 529 struct cs_etm_auxtrace, 530 auxtrace); 531 if (dump_trace) 532 return 0; 533 534 if (!tool->ordered_events) 535 return -EINVAL; 536 537 ret = cs_etm__update_queues(etm); 538 539 if (ret < 0) 540 return ret; 541 542 if (etm->timeless_decoding) 543 return cs_etm__process_timeless_queues(etm, -1); 544 545 return cs_etm__process_queues(etm); 546 } 547 548 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq) 549 { 550 int idx; 551 uintptr_t priv; 552 struct int_node *inode, *tmp; 553 struct cs_etm_traceid_queue *tidq; 554 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 555 556 intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) { 557 priv = (uintptr_t)inode->priv; 558 idx = priv; 559 560 /* Free this traceid_queue from the array */ 561 tidq = etmq->traceid_queues[idx]; 562 thread__zput(tidq->thread); 563 zfree(&tidq->event_buf); 564 zfree(&tidq->last_branch); 565 zfree(&tidq->last_branch_rb); 566 zfree(&tidq->prev_packet); 567 zfree(&tidq->packet); 568 zfree(&tidq); 569 570 /* 571 * Function intlist__remove() removes the inode from the list 572 * and delete the memory associated to it. 573 */ 574 intlist__remove(traceid_queues_list, inode); 575 } 576 577 /* Then the RB tree itself */ 578 intlist__delete(traceid_queues_list); 579 etmq->traceid_queues_list = NULL; 580 581 /* finally free the traceid_queues array */ 582 zfree(&etmq->traceid_queues); 583 } 584 585 static void cs_etm__free_queue(void *priv) 586 { 587 struct cs_etm_queue *etmq = priv; 588 589 if (!etmq) 590 return; 591 592 cs_etm_decoder__free(etmq->decoder); 593 cs_etm__free_traceid_queues(etmq); 594 free(etmq); 595 } 596 597 static void cs_etm__free_events(struct perf_session *session) 598 { 599 unsigned int i; 600 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 601 struct cs_etm_auxtrace, 602 auxtrace); 603 struct auxtrace_queues *queues = &aux->queues; 604 605 for (i = 0; i < queues->nr_queues; i++) { 606 cs_etm__free_queue(queues->queue_array[i].priv); 607 queues->queue_array[i].priv = NULL; 608 } 609 610 auxtrace_queues__free(queues); 611 } 612 613 static void cs_etm__free(struct perf_session *session) 614 { 615 int i; 616 struct int_node *inode, *tmp; 617 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 618 struct cs_etm_auxtrace, 619 auxtrace); 620 cs_etm__free_events(session); 621 session->auxtrace = NULL; 622 623 /* First remove all traceID/metadata nodes for the RB tree */ 624 intlist__for_each_entry_safe(inode, tmp, traceid_list) 625 intlist__remove(traceid_list, inode); 626 /* Then the RB tree itself */ 627 intlist__delete(traceid_list); 628 629 for (i = 0; i < aux->num_cpu; i++) 630 zfree(&aux->metadata[i]); 631 632 thread__zput(aux->unknown_thread); 633 zfree(&aux->metadata); 634 zfree(&aux); 635 } 636 637 static bool cs_etm__evsel_is_auxtrace(struct perf_session *session, 638 struct evsel *evsel) 639 { 640 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 641 struct cs_etm_auxtrace, 642 auxtrace); 643 644 return evsel->core.attr.type == aux->pmu_type; 645 } 646 647 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address) 648 { 649 struct machine *machine; 650 651 machine = etmq->etm->machine; 652 653 if (address >= etmq->etm->kernel_start) { 654 if (machine__is_host(machine)) 655 return PERF_RECORD_MISC_KERNEL; 656 else 657 return PERF_RECORD_MISC_GUEST_KERNEL; 658 } else { 659 if (machine__is_host(machine)) 660 return PERF_RECORD_MISC_USER; 661 else if (perf_guest) 662 return PERF_RECORD_MISC_GUEST_USER; 663 else 664 return PERF_RECORD_MISC_HYPERVISOR; 665 } 666 } 667 668 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, 669 u64 address, size_t size, u8 *buffer) 670 { 671 u8 cpumode; 672 u64 offset; 673 int len; 674 struct thread *thread; 675 struct machine *machine; 676 struct addr_location al; 677 struct cs_etm_traceid_queue *tidq; 678 679 if (!etmq) 680 return 0; 681 682 machine = etmq->etm->machine; 683 cpumode = cs_etm__cpu_mode(etmq, address); 684 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 685 if (!tidq) 686 return 0; 687 688 thread = tidq->thread; 689 if (!thread) { 690 if (cpumode != PERF_RECORD_MISC_KERNEL) 691 return 0; 692 thread = etmq->etm->unknown_thread; 693 } 694 695 if (!thread__find_map(thread, cpumode, address, &al) || !al.map->dso) 696 return 0; 697 698 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR && 699 dso__data_status_seen(al.map->dso, DSO_DATA_STATUS_SEEN_ITRACE)) 700 return 0; 701 702 offset = al.map->map_ip(al.map, address); 703 704 map__load(al.map); 705 706 len = dso__data_read_offset(al.map->dso, machine, offset, buffer, size); 707 708 if (len <= 0) 709 return 0; 710 711 return len; 712 } 713 714 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm) 715 { 716 struct cs_etm_decoder_params d_params; 717 struct cs_etm_trace_params *t_params = NULL; 718 struct cs_etm_queue *etmq; 719 720 etmq = zalloc(sizeof(*etmq)); 721 if (!etmq) 722 return NULL; 723 724 etmq->traceid_queues_list = intlist__new(NULL); 725 if (!etmq->traceid_queues_list) 726 goto out_free; 727 728 /* Use metadata to fill in trace parameters for trace decoder */ 729 t_params = zalloc(sizeof(*t_params) * etm->num_cpu); 730 731 if (!t_params) 732 goto out_free; 733 734 if (cs_etm__init_trace_params(t_params, etm)) 735 goto out_free; 736 737 /* Set decoder parameters to decode trace packets */ 738 if (cs_etm__init_decoder_params(&d_params, etmq, 739 CS_ETM_OPERATION_DECODE)) 740 goto out_free; 741 742 etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params); 743 744 if (!etmq->decoder) 745 goto out_free; 746 747 /* 748 * Register a function to handle all memory accesses required by 749 * the trace decoder library. 750 */ 751 if (cs_etm_decoder__add_mem_access_cb(etmq->decoder, 752 0x0L, ((u64) -1L), 753 cs_etm__mem_access)) 754 goto out_free_decoder; 755 756 zfree(&t_params); 757 return etmq; 758 759 out_free_decoder: 760 cs_etm_decoder__free(etmq->decoder); 761 out_free: 762 intlist__delete(etmq->traceid_queues_list); 763 free(etmq); 764 765 return NULL; 766 } 767 768 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, 769 struct auxtrace_queue *queue, 770 unsigned int queue_nr) 771 { 772 int ret = 0; 773 unsigned int cs_queue_nr; 774 u8 trace_chan_id; 775 u64 timestamp; 776 struct cs_etm_queue *etmq = queue->priv; 777 778 if (list_empty(&queue->head) || etmq) 779 goto out; 780 781 etmq = cs_etm__alloc_queue(etm); 782 783 if (!etmq) { 784 ret = -ENOMEM; 785 goto out; 786 } 787 788 queue->priv = etmq; 789 etmq->etm = etm; 790 etmq->queue_nr = queue_nr; 791 etmq->offset = 0; 792 793 if (etm->timeless_decoding) 794 goto out; 795 796 /* 797 * We are under a CPU-wide trace scenario. As such we need to know 798 * when the code that generated the traces started to execute so that 799 * it can be correlated with execution on other CPUs. So we get a 800 * handle on the beginning of traces and decode until we find a 801 * timestamp. The timestamp is then added to the auxtrace min heap 802 * in order to know what nibble (of all the etmqs) to decode first. 803 */ 804 while (1) { 805 /* 806 * Fetch an aux_buffer from this etmq. Bail if no more 807 * blocks or an error has been encountered. 808 */ 809 ret = cs_etm__get_data_block(etmq); 810 if (ret <= 0) 811 goto out; 812 813 /* 814 * Run decoder on the trace block. The decoder will stop when 815 * encountering a timestamp, a full packet queue or the end of 816 * trace for that block. 817 */ 818 ret = cs_etm__decode_data_block(etmq); 819 if (ret) 820 goto out; 821 822 /* 823 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all 824 * the timestamp calculation for us. 825 */ 826 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); 827 828 /* We found a timestamp, no need to continue. */ 829 if (timestamp) 830 break; 831 832 /* 833 * We didn't find a timestamp so empty all the traceid packet 834 * queues before looking for another timestamp packet, either 835 * in the current data block or a new one. Packets that were 836 * just decoded are useless since no timestamp has been 837 * associated with them. As such simply discard them. 838 */ 839 cs_etm__clear_all_packet_queues(etmq); 840 } 841 842 /* 843 * We have a timestamp. Add it to the min heap to reflect when 844 * instructions conveyed by the range packets of this traceID queue 845 * started to execute. Once the same has been done for all the traceID 846 * queues of each etmq, redenring and decoding can start in 847 * chronological order. 848 * 849 * Note that packets decoded above are still in the traceID's packet 850 * queue and will be processed in cs_etm__process_queues(). 851 */ 852 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); 853 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp); 854 out: 855 return ret; 856 } 857 858 static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm) 859 { 860 unsigned int i; 861 int ret; 862 863 if (!etm->kernel_start) 864 etm->kernel_start = machine__kernel_start(etm->machine); 865 866 for (i = 0; i < etm->queues.nr_queues; i++) { 867 ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i); 868 if (ret) 869 return ret; 870 } 871 872 return 0; 873 } 874 875 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm) 876 { 877 if (etm->queues.new_data) { 878 etm->queues.new_data = false; 879 return cs_etm__setup_queues(etm); 880 } 881 882 return 0; 883 } 884 885 static inline 886 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq, 887 struct cs_etm_traceid_queue *tidq) 888 { 889 struct branch_stack *bs_src = tidq->last_branch_rb; 890 struct branch_stack *bs_dst = tidq->last_branch; 891 size_t nr = 0; 892 893 /* 894 * Set the number of records before early exit: ->nr is used to 895 * determine how many branches to copy from ->entries. 896 */ 897 bs_dst->nr = bs_src->nr; 898 899 /* 900 * Early exit when there is nothing to copy. 901 */ 902 if (!bs_src->nr) 903 return; 904 905 /* 906 * As bs_src->entries is a circular buffer, we need to copy from it in 907 * two steps. First, copy the branches from the most recently inserted 908 * branch ->last_branch_pos until the end of bs_src->entries buffer. 909 */ 910 nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos; 911 memcpy(&bs_dst->entries[0], 912 &bs_src->entries[tidq->last_branch_pos], 913 sizeof(struct branch_entry) * nr); 914 915 /* 916 * If we wrapped around at least once, the branches from the beginning 917 * of the bs_src->entries buffer and until the ->last_branch_pos element 918 * are older valid branches: copy them over. The total number of 919 * branches copied over will be equal to the number of branches asked by 920 * the user in last_branch_sz. 921 */ 922 if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) { 923 memcpy(&bs_dst->entries[nr], 924 &bs_src->entries[0], 925 sizeof(struct branch_entry) * tidq->last_branch_pos); 926 } 927 } 928 929 static inline 930 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq) 931 { 932 tidq->last_branch_pos = 0; 933 tidq->last_branch_rb->nr = 0; 934 } 935 936 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, 937 u8 trace_chan_id, u64 addr) 938 { 939 u8 instrBytes[2]; 940 941 cs_etm__mem_access(etmq, trace_chan_id, addr, 942 ARRAY_SIZE(instrBytes), instrBytes); 943 /* 944 * T32 instruction size is indicated by bits[15:11] of the first 945 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111 946 * denote a 32-bit instruction. 947 */ 948 return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2; 949 } 950 951 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet) 952 { 953 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */ 954 if (packet->sample_type == CS_ETM_DISCONTINUITY) 955 return 0; 956 957 return packet->start_addr; 958 } 959 960 static inline 961 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet) 962 { 963 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */ 964 if (packet->sample_type == CS_ETM_DISCONTINUITY) 965 return 0; 966 967 return packet->end_addr - packet->last_instr_size; 968 } 969 970 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq, 971 u64 trace_chan_id, 972 const struct cs_etm_packet *packet, 973 u64 offset) 974 { 975 if (packet->isa == CS_ETM_ISA_T32) { 976 u64 addr = packet->start_addr; 977 978 while (offset) { 979 addr += cs_etm__t32_instr_size(etmq, 980 trace_chan_id, addr); 981 offset--; 982 } 983 return addr; 984 } 985 986 /* Assume a 4 byte instruction size (A32/A64) */ 987 return packet->start_addr + offset * 4; 988 } 989 990 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq, 991 struct cs_etm_traceid_queue *tidq) 992 { 993 struct branch_stack *bs = tidq->last_branch_rb; 994 struct branch_entry *be; 995 996 /* 997 * The branches are recorded in a circular buffer in reverse 998 * chronological order: we start recording from the last element of the 999 * buffer down. After writing the first element of the stack, move the 1000 * insert position back to the end of the buffer. 1001 */ 1002 if (!tidq->last_branch_pos) 1003 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz; 1004 1005 tidq->last_branch_pos -= 1; 1006 1007 be = &bs->entries[tidq->last_branch_pos]; 1008 be->from = cs_etm__last_executed_instr(tidq->prev_packet); 1009 be->to = cs_etm__first_executed_instr(tidq->packet); 1010 /* No support for mispredict */ 1011 be->flags.mispred = 0; 1012 be->flags.predicted = 1; 1013 1014 /* 1015 * Increment bs->nr until reaching the number of last branches asked by 1016 * the user on the command line. 1017 */ 1018 if (bs->nr < etmq->etm->synth_opts.last_branch_sz) 1019 bs->nr += 1; 1020 } 1021 1022 static int cs_etm__inject_event(union perf_event *event, 1023 struct perf_sample *sample, u64 type) 1024 { 1025 event->header.size = perf_event__sample_event_size(sample, type, 0); 1026 return perf_event__synthesize_sample(event, type, 0, sample); 1027 } 1028 1029 1030 static int 1031 cs_etm__get_trace(struct cs_etm_queue *etmq) 1032 { 1033 struct auxtrace_buffer *aux_buffer = etmq->buffer; 1034 struct auxtrace_buffer *old_buffer = aux_buffer; 1035 struct auxtrace_queue *queue; 1036 1037 queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; 1038 1039 aux_buffer = auxtrace_buffer__next(queue, aux_buffer); 1040 1041 /* If no more data, drop the previous auxtrace_buffer and return */ 1042 if (!aux_buffer) { 1043 if (old_buffer) 1044 auxtrace_buffer__drop_data(old_buffer); 1045 etmq->buf_len = 0; 1046 return 0; 1047 } 1048 1049 etmq->buffer = aux_buffer; 1050 1051 /* If the aux_buffer doesn't have data associated, try to load it */ 1052 if (!aux_buffer->data) { 1053 /* get the file desc associated with the perf data file */ 1054 int fd = perf_data__fd(etmq->etm->session->data); 1055 1056 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd); 1057 if (!aux_buffer->data) 1058 return -ENOMEM; 1059 } 1060 1061 /* If valid, drop the previous buffer */ 1062 if (old_buffer) 1063 auxtrace_buffer__drop_data(old_buffer); 1064 1065 etmq->buf_used = 0; 1066 etmq->buf_len = aux_buffer->size; 1067 etmq->buf = aux_buffer->data; 1068 1069 return etmq->buf_len; 1070 } 1071 1072 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm, 1073 struct cs_etm_traceid_queue *tidq) 1074 { 1075 if ((!tidq->thread) && (tidq->tid != -1)) 1076 tidq->thread = machine__find_thread(etm->machine, -1, 1077 tidq->tid); 1078 1079 if (tidq->thread) 1080 tidq->pid = tidq->thread->pid_; 1081 } 1082 1083 int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq, 1084 pid_t tid, u8 trace_chan_id) 1085 { 1086 int cpu, err = -EINVAL; 1087 struct cs_etm_auxtrace *etm = etmq->etm; 1088 struct cs_etm_traceid_queue *tidq; 1089 1090 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 1091 if (!tidq) 1092 return err; 1093 1094 if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0) 1095 return err; 1096 1097 err = machine__set_current_tid(etm->machine, cpu, tid, tid); 1098 if (err) 1099 return err; 1100 1101 tidq->tid = tid; 1102 thread__zput(tidq->thread); 1103 1104 cs_etm__set_pid_tid_cpu(etm, tidq); 1105 return 0; 1106 } 1107 1108 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq) 1109 { 1110 return !!etmq->etm->timeless_decoding; 1111 } 1112 1113 static void cs_etm__copy_insn(struct cs_etm_queue *etmq, 1114 u64 trace_chan_id, 1115 const struct cs_etm_packet *packet, 1116 struct perf_sample *sample) 1117 { 1118 /* 1119 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY 1120 * packet, so directly bail out with 'insn_len' = 0. 1121 */ 1122 if (packet->sample_type == CS_ETM_DISCONTINUITY) { 1123 sample->insn_len = 0; 1124 return; 1125 } 1126 1127 /* 1128 * T32 instruction size might be 32-bit or 16-bit, decide by calling 1129 * cs_etm__t32_instr_size(). 1130 */ 1131 if (packet->isa == CS_ETM_ISA_T32) 1132 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id, 1133 sample->ip); 1134 /* Otherwise, A64 and A32 instruction size are always 32-bit. */ 1135 else 1136 sample->insn_len = 4; 1137 1138 cs_etm__mem_access(etmq, trace_chan_id, sample->ip, 1139 sample->insn_len, (void *)sample->insn); 1140 } 1141 1142 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, 1143 struct cs_etm_traceid_queue *tidq, 1144 u64 addr, u64 period) 1145 { 1146 int ret = 0; 1147 struct cs_etm_auxtrace *etm = etmq->etm; 1148 union perf_event *event = tidq->event_buf; 1149 struct perf_sample sample = {.ip = 0,}; 1150 1151 event->sample.header.type = PERF_RECORD_SAMPLE; 1152 event->sample.header.misc = cs_etm__cpu_mode(etmq, addr); 1153 event->sample.header.size = sizeof(struct perf_event_header); 1154 1155 sample.ip = addr; 1156 sample.pid = tidq->pid; 1157 sample.tid = tidq->tid; 1158 sample.id = etmq->etm->instructions_id; 1159 sample.stream_id = etmq->etm->instructions_id; 1160 sample.period = period; 1161 sample.cpu = tidq->packet->cpu; 1162 sample.flags = tidq->prev_packet->flags; 1163 sample.cpumode = event->sample.header.misc; 1164 1165 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample); 1166 1167 if (etm->synth_opts.last_branch) 1168 sample.branch_stack = tidq->last_branch; 1169 1170 if (etm->synth_opts.inject) { 1171 ret = cs_etm__inject_event(event, &sample, 1172 etm->instructions_sample_type); 1173 if (ret) 1174 return ret; 1175 } 1176 1177 ret = perf_session__deliver_synth_event(etm->session, event, &sample); 1178 1179 if (ret) 1180 pr_err( 1181 "CS ETM Trace: failed to deliver instruction event, error %d\n", 1182 ret); 1183 1184 return ret; 1185 } 1186 1187 /* 1188 * The cs etm packet encodes an instruction range between a branch target 1189 * and the next taken branch. Generate sample accordingly. 1190 */ 1191 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, 1192 struct cs_etm_traceid_queue *tidq) 1193 { 1194 int ret = 0; 1195 struct cs_etm_auxtrace *etm = etmq->etm; 1196 struct perf_sample sample = {.ip = 0,}; 1197 union perf_event *event = tidq->event_buf; 1198 struct dummy_branch_stack { 1199 u64 nr; 1200 u64 hw_idx; 1201 struct branch_entry entries; 1202 } dummy_bs; 1203 u64 ip; 1204 1205 ip = cs_etm__last_executed_instr(tidq->prev_packet); 1206 1207 event->sample.header.type = PERF_RECORD_SAMPLE; 1208 event->sample.header.misc = cs_etm__cpu_mode(etmq, ip); 1209 event->sample.header.size = sizeof(struct perf_event_header); 1210 1211 sample.ip = ip; 1212 sample.pid = tidq->pid; 1213 sample.tid = tidq->tid; 1214 sample.addr = cs_etm__first_executed_instr(tidq->packet); 1215 sample.id = etmq->etm->branches_id; 1216 sample.stream_id = etmq->etm->branches_id; 1217 sample.period = 1; 1218 sample.cpu = tidq->packet->cpu; 1219 sample.flags = tidq->prev_packet->flags; 1220 sample.cpumode = event->sample.header.misc; 1221 1222 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet, 1223 &sample); 1224 1225 /* 1226 * perf report cannot handle events without a branch stack 1227 */ 1228 if (etm->synth_opts.last_branch) { 1229 dummy_bs = (struct dummy_branch_stack){ 1230 .nr = 1, 1231 .hw_idx = -1ULL, 1232 .entries = { 1233 .from = sample.ip, 1234 .to = sample.addr, 1235 }, 1236 }; 1237 sample.branch_stack = (struct branch_stack *)&dummy_bs; 1238 } 1239 1240 if (etm->synth_opts.inject) { 1241 ret = cs_etm__inject_event(event, &sample, 1242 etm->branches_sample_type); 1243 if (ret) 1244 return ret; 1245 } 1246 1247 ret = perf_session__deliver_synth_event(etm->session, event, &sample); 1248 1249 if (ret) 1250 pr_err( 1251 "CS ETM Trace: failed to deliver instruction event, error %d\n", 1252 ret); 1253 1254 return ret; 1255 } 1256 1257 struct cs_etm_synth { 1258 struct perf_tool dummy_tool; 1259 struct perf_session *session; 1260 }; 1261 1262 static int cs_etm__event_synth(struct perf_tool *tool, 1263 union perf_event *event, 1264 struct perf_sample *sample __maybe_unused, 1265 struct machine *machine __maybe_unused) 1266 { 1267 struct cs_etm_synth *cs_etm_synth = 1268 container_of(tool, struct cs_etm_synth, dummy_tool); 1269 1270 return perf_session__deliver_synth_event(cs_etm_synth->session, 1271 event, NULL); 1272 } 1273 1274 static int cs_etm__synth_event(struct perf_session *session, 1275 struct perf_event_attr *attr, u64 id) 1276 { 1277 struct cs_etm_synth cs_etm_synth; 1278 1279 memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth)); 1280 cs_etm_synth.session = session; 1281 1282 return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1, 1283 &id, cs_etm__event_synth); 1284 } 1285 1286 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm, 1287 struct perf_session *session) 1288 { 1289 struct evlist *evlist = session->evlist; 1290 struct evsel *evsel; 1291 struct perf_event_attr attr; 1292 bool found = false; 1293 u64 id; 1294 int err; 1295 1296 evlist__for_each_entry(evlist, evsel) { 1297 if (evsel->core.attr.type == etm->pmu_type) { 1298 found = true; 1299 break; 1300 } 1301 } 1302 1303 if (!found) { 1304 pr_debug("No selected events with CoreSight Trace data\n"); 1305 return 0; 1306 } 1307 1308 memset(&attr, 0, sizeof(struct perf_event_attr)); 1309 attr.size = sizeof(struct perf_event_attr); 1310 attr.type = PERF_TYPE_HARDWARE; 1311 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK; 1312 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | 1313 PERF_SAMPLE_PERIOD; 1314 if (etm->timeless_decoding) 1315 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; 1316 else 1317 attr.sample_type |= PERF_SAMPLE_TIME; 1318 1319 attr.exclude_user = evsel->core.attr.exclude_user; 1320 attr.exclude_kernel = evsel->core.attr.exclude_kernel; 1321 attr.exclude_hv = evsel->core.attr.exclude_hv; 1322 attr.exclude_host = evsel->core.attr.exclude_host; 1323 attr.exclude_guest = evsel->core.attr.exclude_guest; 1324 attr.sample_id_all = evsel->core.attr.sample_id_all; 1325 attr.read_format = evsel->core.attr.read_format; 1326 1327 /* create new id val to be a fixed offset from evsel id */ 1328 id = evsel->core.id[0] + 1000000000; 1329 1330 if (!id) 1331 id = 1; 1332 1333 if (etm->synth_opts.branches) { 1334 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS; 1335 attr.sample_period = 1; 1336 attr.sample_type |= PERF_SAMPLE_ADDR; 1337 err = cs_etm__synth_event(session, &attr, id); 1338 if (err) 1339 return err; 1340 etm->sample_branches = true; 1341 etm->branches_sample_type = attr.sample_type; 1342 etm->branches_id = id; 1343 id += 1; 1344 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR; 1345 } 1346 1347 if (etm->synth_opts.last_branch) { 1348 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; 1349 /* 1350 * We don't use the hardware index, but the sample generation 1351 * code uses the new format branch_stack with this field, 1352 * so the event attributes must indicate that it's present. 1353 */ 1354 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; 1355 } 1356 1357 if (etm->synth_opts.instructions) { 1358 attr.config = PERF_COUNT_HW_INSTRUCTIONS; 1359 attr.sample_period = etm->synth_opts.period; 1360 etm->instructions_sample_period = attr.sample_period; 1361 err = cs_etm__synth_event(session, &attr, id); 1362 if (err) 1363 return err; 1364 etm->sample_instructions = true; 1365 etm->instructions_sample_type = attr.sample_type; 1366 etm->instructions_id = id; 1367 id += 1; 1368 } 1369 1370 return 0; 1371 } 1372 1373 static int cs_etm__sample(struct cs_etm_queue *etmq, 1374 struct cs_etm_traceid_queue *tidq) 1375 { 1376 struct cs_etm_auxtrace *etm = etmq->etm; 1377 int ret; 1378 u8 trace_chan_id = tidq->trace_chan_id; 1379 u64 instrs_prev; 1380 1381 /* Get instructions remainder from previous packet */ 1382 instrs_prev = tidq->period_instructions; 1383 1384 tidq->period_instructions += tidq->packet->instr_count; 1385 1386 /* 1387 * Record a branch when the last instruction in 1388 * PREV_PACKET is a branch. 1389 */ 1390 if (etm->synth_opts.last_branch && 1391 tidq->prev_packet->sample_type == CS_ETM_RANGE && 1392 tidq->prev_packet->last_instr_taken_branch) 1393 cs_etm__update_last_branch_rb(etmq, tidq); 1394 1395 if (etm->sample_instructions && 1396 tidq->period_instructions >= etm->instructions_sample_period) { 1397 /* 1398 * Emit instruction sample periodically 1399 * TODO: allow period to be defined in cycles and clock time 1400 */ 1401 1402 /* 1403 * Below diagram demonstrates the instruction samples 1404 * generation flows: 1405 * 1406 * Instrs Instrs Instrs Instrs 1407 * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3) 1408 * | | | | 1409 * V V V V 1410 * -------------------------------------------------- 1411 * ^ ^ 1412 * | | 1413 * Period Period 1414 * instructions(Pi) instructions(Pi') 1415 * 1416 * | | 1417 * \---------------- -----------------/ 1418 * V 1419 * tidq->packet->instr_count 1420 * 1421 * Instrs Sample(n...) are the synthesised samples occurring 1422 * every etm->instructions_sample_period instructions - as 1423 * defined on the perf command line. Sample(n) is being the 1424 * last sample before the current etm packet, n+1 to n+3 1425 * samples are generated from the current etm packet. 1426 * 1427 * tidq->packet->instr_count represents the number of 1428 * instructions in the current etm packet. 1429 * 1430 * Period instructions (Pi) contains the the number of 1431 * instructions executed after the sample point(n) from the 1432 * previous etm packet. This will always be less than 1433 * etm->instructions_sample_period. 1434 * 1435 * When generate new samples, it combines with two parts 1436 * instructions, one is the tail of the old packet and another 1437 * is the head of the new coming packet, to generate 1438 * sample(n+1); sample(n+2) and sample(n+3) consume the 1439 * instructions with sample period. After sample(n+3), the rest 1440 * instructions will be used by later packet and it is assigned 1441 * to tidq->period_instructions for next round calculation. 1442 */ 1443 1444 /* 1445 * Get the initial offset into the current packet instructions; 1446 * entry conditions ensure that instrs_prev is less than 1447 * etm->instructions_sample_period. 1448 */ 1449 u64 offset = etm->instructions_sample_period - instrs_prev; 1450 u64 addr; 1451 1452 /* Prepare last branches for instruction sample */ 1453 if (etm->synth_opts.last_branch) 1454 cs_etm__copy_last_branch_rb(etmq, tidq); 1455 1456 while (tidq->period_instructions >= 1457 etm->instructions_sample_period) { 1458 /* 1459 * Calculate the address of the sampled instruction (-1 1460 * as sample is reported as though instruction has just 1461 * been executed, but PC has not advanced to next 1462 * instruction) 1463 */ 1464 addr = cs_etm__instr_addr(etmq, trace_chan_id, 1465 tidq->packet, offset - 1); 1466 ret = cs_etm__synth_instruction_sample( 1467 etmq, tidq, addr, 1468 etm->instructions_sample_period); 1469 if (ret) 1470 return ret; 1471 1472 offset += etm->instructions_sample_period; 1473 tidq->period_instructions -= 1474 etm->instructions_sample_period; 1475 } 1476 } 1477 1478 if (etm->sample_branches) { 1479 bool generate_sample = false; 1480 1481 /* Generate sample for tracing on packet */ 1482 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY) 1483 generate_sample = true; 1484 1485 /* Generate sample for branch taken packet */ 1486 if (tidq->prev_packet->sample_type == CS_ETM_RANGE && 1487 tidq->prev_packet->last_instr_taken_branch) 1488 generate_sample = true; 1489 1490 if (generate_sample) { 1491 ret = cs_etm__synth_branch_sample(etmq, tidq); 1492 if (ret) 1493 return ret; 1494 } 1495 } 1496 1497 cs_etm__packet_swap(etm, tidq); 1498 1499 return 0; 1500 } 1501 1502 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq) 1503 { 1504 /* 1505 * When the exception packet is inserted, whether the last instruction 1506 * in previous range packet is taken branch or not, we need to force 1507 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures 1508 * to generate branch sample for the instruction range before the 1509 * exception is trapped to kernel or before the exception returning. 1510 * 1511 * The exception packet includes the dummy address values, so don't 1512 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful 1513 * for generating instruction and branch samples. 1514 */ 1515 if (tidq->prev_packet->sample_type == CS_ETM_RANGE) 1516 tidq->prev_packet->last_instr_taken_branch = true; 1517 1518 return 0; 1519 } 1520 1521 static int cs_etm__flush(struct cs_etm_queue *etmq, 1522 struct cs_etm_traceid_queue *tidq) 1523 { 1524 int err = 0; 1525 struct cs_etm_auxtrace *etm = etmq->etm; 1526 1527 /* Handle start tracing packet */ 1528 if (tidq->prev_packet->sample_type == CS_ETM_EMPTY) 1529 goto swap_packet; 1530 1531 if (etmq->etm->synth_opts.last_branch && 1532 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 1533 u64 addr; 1534 1535 /* Prepare last branches for instruction sample */ 1536 cs_etm__copy_last_branch_rb(etmq, tidq); 1537 1538 /* 1539 * Generate a last branch event for the branches left in the 1540 * circular buffer at the end of the trace. 1541 * 1542 * Use the address of the end of the last reported execution 1543 * range 1544 */ 1545 addr = cs_etm__last_executed_instr(tidq->prev_packet); 1546 1547 err = cs_etm__synth_instruction_sample( 1548 etmq, tidq, addr, 1549 tidq->period_instructions); 1550 if (err) 1551 return err; 1552 1553 tidq->period_instructions = 0; 1554 1555 } 1556 1557 if (etm->sample_branches && 1558 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 1559 err = cs_etm__synth_branch_sample(etmq, tidq); 1560 if (err) 1561 return err; 1562 } 1563 1564 swap_packet: 1565 cs_etm__packet_swap(etm, tidq); 1566 1567 /* Reset last branches after flush the trace */ 1568 if (etm->synth_opts.last_branch) 1569 cs_etm__reset_last_branch_rb(tidq); 1570 1571 return err; 1572 } 1573 1574 static int cs_etm__end_block(struct cs_etm_queue *etmq, 1575 struct cs_etm_traceid_queue *tidq) 1576 { 1577 int err; 1578 1579 /* 1580 * It has no new packet coming and 'etmq->packet' contains the stale 1581 * packet which was set at the previous time with packets swapping; 1582 * so skip to generate branch sample to avoid stale packet. 1583 * 1584 * For this case only flush branch stack and generate a last branch 1585 * event for the branches left in the circular buffer at the end of 1586 * the trace. 1587 */ 1588 if (etmq->etm->synth_opts.last_branch && 1589 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 1590 u64 addr; 1591 1592 /* Prepare last branches for instruction sample */ 1593 cs_etm__copy_last_branch_rb(etmq, tidq); 1594 1595 /* 1596 * Use the address of the end of the last reported execution 1597 * range. 1598 */ 1599 addr = cs_etm__last_executed_instr(tidq->prev_packet); 1600 1601 err = cs_etm__synth_instruction_sample( 1602 etmq, tidq, addr, 1603 tidq->period_instructions); 1604 if (err) 1605 return err; 1606 1607 tidq->period_instructions = 0; 1608 } 1609 1610 return 0; 1611 } 1612 /* 1613 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue 1614 * if need be. 1615 * Returns: < 0 if error 1616 * = 0 if no more auxtrace_buffer to read 1617 * > 0 if the current buffer isn't empty yet 1618 */ 1619 static int cs_etm__get_data_block(struct cs_etm_queue *etmq) 1620 { 1621 int ret; 1622 1623 if (!etmq->buf_len) { 1624 ret = cs_etm__get_trace(etmq); 1625 if (ret <= 0) 1626 return ret; 1627 /* 1628 * We cannot assume consecutive blocks in the data file 1629 * are contiguous, reset the decoder to force re-sync. 1630 */ 1631 ret = cs_etm_decoder__reset(etmq->decoder); 1632 if (ret) 1633 return ret; 1634 } 1635 1636 return etmq->buf_len; 1637 } 1638 1639 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, 1640 struct cs_etm_packet *packet, 1641 u64 end_addr) 1642 { 1643 /* Initialise to keep compiler happy */ 1644 u16 instr16 = 0; 1645 u32 instr32 = 0; 1646 u64 addr; 1647 1648 switch (packet->isa) { 1649 case CS_ETM_ISA_T32: 1650 /* 1651 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247: 1652 * 1653 * b'15 b'8 1654 * +-----------------+--------+ 1655 * | 1 1 0 1 1 1 1 1 | imm8 | 1656 * +-----------------+--------+ 1657 * 1658 * According to the specifiction, it only defines SVC for T32 1659 * with 16 bits instruction and has no definition for 32bits; 1660 * so below only read 2 bytes as instruction size for T32. 1661 */ 1662 addr = end_addr - 2; 1663 cs_etm__mem_access(etmq, trace_chan_id, addr, 1664 sizeof(instr16), (u8 *)&instr16); 1665 if ((instr16 & 0xFF00) == 0xDF00) 1666 return true; 1667 1668 break; 1669 case CS_ETM_ISA_A32: 1670 /* 1671 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247: 1672 * 1673 * b'31 b'28 b'27 b'24 1674 * +---------+---------+-------------------------+ 1675 * | !1111 | 1 1 1 1 | imm24 | 1676 * +---------+---------+-------------------------+ 1677 */ 1678 addr = end_addr - 4; 1679 cs_etm__mem_access(etmq, trace_chan_id, addr, 1680 sizeof(instr32), (u8 *)&instr32); 1681 if ((instr32 & 0x0F000000) == 0x0F000000 && 1682 (instr32 & 0xF0000000) != 0xF0000000) 1683 return true; 1684 1685 break; 1686 case CS_ETM_ISA_A64: 1687 /* 1688 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294: 1689 * 1690 * b'31 b'21 b'4 b'0 1691 * +-----------------------+---------+-----------+ 1692 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 | 1693 * +-----------------------+---------+-----------+ 1694 */ 1695 addr = end_addr - 4; 1696 cs_etm__mem_access(etmq, trace_chan_id, addr, 1697 sizeof(instr32), (u8 *)&instr32); 1698 if ((instr32 & 0xFFE0001F) == 0xd4000001) 1699 return true; 1700 1701 break; 1702 case CS_ETM_ISA_UNKNOWN: 1703 default: 1704 break; 1705 } 1706 1707 return false; 1708 } 1709 1710 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, 1711 struct cs_etm_traceid_queue *tidq, u64 magic) 1712 { 1713 u8 trace_chan_id = tidq->trace_chan_id; 1714 struct cs_etm_packet *packet = tidq->packet; 1715 struct cs_etm_packet *prev_packet = tidq->prev_packet; 1716 1717 if (magic == __perf_cs_etmv3_magic) 1718 if (packet->exception_number == CS_ETMV3_EXC_SVC) 1719 return true; 1720 1721 /* 1722 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and 1723 * HVC cases; need to check if it's SVC instruction based on 1724 * packet address. 1725 */ 1726 if (magic == __perf_cs_etmv4_magic) { 1727 if (packet->exception_number == CS_ETMV4_EXC_CALL && 1728 cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, 1729 prev_packet->end_addr)) 1730 return true; 1731 } 1732 1733 return false; 1734 } 1735 1736 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq, 1737 u64 magic) 1738 { 1739 struct cs_etm_packet *packet = tidq->packet; 1740 1741 if (magic == __perf_cs_etmv3_magic) 1742 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT || 1743 packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT || 1744 packet->exception_number == CS_ETMV3_EXC_PE_RESET || 1745 packet->exception_number == CS_ETMV3_EXC_IRQ || 1746 packet->exception_number == CS_ETMV3_EXC_FIQ) 1747 return true; 1748 1749 if (magic == __perf_cs_etmv4_magic) 1750 if (packet->exception_number == CS_ETMV4_EXC_RESET || 1751 packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT || 1752 packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR || 1753 packet->exception_number == CS_ETMV4_EXC_INST_DEBUG || 1754 packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG || 1755 packet->exception_number == CS_ETMV4_EXC_IRQ || 1756 packet->exception_number == CS_ETMV4_EXC_FIQ) 1757 return true; 1758 1759 return false; 1760 } 1761 1762 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, 1763 struct cs_etm_traceid_queue *tidq, 1764 u64 magic) 1765 { 1766 u8 trace_chan_id = tidq->trace_chan_id; 1767 struct cs_etm_packet *packet = tidq->packet; 1768 struct cs_etm_packet *prev_packet = tidq->prev_packet; 1769 1770 if (magic == __perf_cs_etmv3_magic) 1771 if (packet->exception_number == CS_ETMV3_EXC_SMC || 1772 packet->exception_number == CS_ETMV3_EXC_HYP || 1773 packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE || 1774 packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR || 1775 packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT || 1776 packet->exception_number == CS_ETMV3_EXC_DATA_FAULT || 1777 packet->exception_number == CS_ETMV3_EXC_GENERIC) 1778 return true; 1779 1780 if (magic == __perf_cs_etmv4_magic) { 1781 if (packet->exception_number == CS_ETMV4_EXC_TRAP || 1782 packet->exception_number == CS_ETMV4_EXC_ALIGNMENT || 1783 packet->exception_number == CS_ETMV4_EXC_INST_FAULT || 1784 packet->exception_number == CS_ETMV4_EXC_DATA_FAULT) 1785 return true; 1786 1787 /* 1788 * For CS_ETMV4_EXC_CALL, except SVC other instructions 1789 * (SMC, HVC) are taken as sync exceptions. 1790 */ 1791 if (packet->exception_number == CS_ETMV4_EXC_CALL && 1792 !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, 1793 prev_packet->end_addr)) 1794 return true; 1795 1796 /* 1797 * ETMv4 has 5 bits for exception number; if the numbers 1798 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ] 1799 * they are implementation defined exceptions. 1800 * 1801 * For this case, simply take it as sync exception. 1802 */ 1803 if (packet->exception_number > CS_ETMV4_EXC_FIQ && 1804 packet->exception_number <= CS_ETMV4_EXC_END) 1805 return true; 1806 } 1807 1808 return false; 1809 } 1810 1811 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, 1812 struct cs_etm_traceid_queue *tidq) 1813 { 1814 struct cs_etm_packet *packet = tidq->packet; 1815 struct cs_etm_packet *prev_packet = tidq->prev_packet; 1816 u8 trace_chan_id = tidq->trace_chan_id; 1817 u64 magic; 1818 int ret; 1819 1820 switch (packet->sample_type) { 1821 case CS_ETM_RANGE: 1822 /* 1823 * Immediate branch instruction without neither link nor 1824 * return flag, it's normal branch instruction within 1825 * the function. 1826 */ 1827 if (packet->last_instr_type == OCSD_INSTR_BR && 1828 packet->last_instr_subtype == OCSD_S_INSTR_NONE) { 1829 packet->flags = PERF_IP_FLAG_BRANCH; 1830 1831 if (packet->last_instr_cond) 1832 packet->flags |= PERF_IP_FLAG_CONDITIONAL; 1833 } 1834 1835 /* 1836 * Immediate branch instruction with link (e.g. BL), this is 1837 * branch instruction for function call. 1838 */ 1839 if (packet->last_instr_type == OCSD_INSTR_BR && 1840 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) 1841 packet->flags = PERF_IP_FLAG_BRANCH | 1842 PERF_IP_FLAG_CALL; 1843 1844 /* 1845 * Indirect branch instruction with link (e.g. BLR), this is 1846 * branch instruction for function call. 1847 */ 1848 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 1849 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) 1850 packet->flags = PERF_IP_FLAG_BRANCH | 1851 PERF_IP_FLAG_CALL; 1852 1853 /* 1854 * Indirect branch instruction with subtype of 1855 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for 1856 * function return for A32/T32. 1857 */ 1858 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 1859 packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET) 1860 packet->flags = PERF_IP_FLAG_BRANCH | 1861 PERF_IP_FLAG_RETURN; 1862 1863 /* 1864 * Indirect branch instruction without link (e.g. BR), usually 1865 * this is used for function return, especially for functions 1866 * within dynamic link lib. 1867 */ 1868 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 1869 packet->last_instr_subtype == OCSD_S_INSTR_NONE) 1870 packet->flags = PERF_IP_FLAG_BRANCH | 1871 PERF_IP_FLAG_RETURN; 1872 1873 /* Return instruction for function return. */ 1874 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 1875 packet->last_instr_subtype == OCSD_S_INSTR_V8_RET) 1876 packet->flags = PERF_IP_FLAG_BRANCH | 1877 PERF_IP_FLAG_RETURN; 1878 1879 /* 1880 * Decoder might insert a discontinuity in the middle of 1881 * instruction packets, fixup prev_packet with flag 1882 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace. 1883 */ 1884 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY) 1885 prev_packet->flags |= PERF_IP_FLAG_BRANCH | 1886 PERF_IP_FLAG_TRACE_BEGIN; 1887 1888 /* 1889 * If the previous packet is an exception return packet 1890 * and the return address just follows SVC instuction, 1891 * it needs to calibrate the previous packet sample flags 1892 * as PERF_IP_FLAG_SYSCALLRET. 1893 */ 1894 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH | 1895 PERF_IP_FLAG_RETURN | 1896 PERF_IP_FLAG_INTERRUPT) && 1897 cs_etm__is_svc_instr(etmq, trace_chan_id, 1898 packet, packet->start_addr)) 1899 prev_packet->flags = PERF_IP_FLAG_BRANCH | 1900 PERF_IP_FLAG_RETURN | 1901 PERF_IP_FLAG_SYSCALLRET; 1902 break; 1903 case CS_ETM_DISCONTINUITY: 1904 /* 1905 * The trace is discontinuous, if the previous packet is 1906 * instruction packet, set flag PERF_IP_FLAG_TRACE_END 1907 * for previous packet. 1908 */ 1909 if (prev_packet->sample_type == CS_ETM_RANGE) 1910 prev_packet->flags |= PERF_IP_FLAG_BRANCH | 1911 PERF_IP_FLAG_TRACE_END; 1912 break; 1913 case CS_ETM_EXCEPTION: 1914 ret = cs_etm__get_magic(packet->trace_chan_id, &magic); 1915 if (ret) 1916 return ret; 1917 1918 /* The exception is for system call. */ 1919 if (cs_etm__is_syscall(etmq, tidq, magic)) 1920 packet->flags = PERF_IP_FLAG_BRANCH | 1921 PERF_IP_FLAG_CALL | 1922 PERF_IP_FLAG_SYSCALLRET; 1923 /* 1924 * The exceptions are triggered by external signals from bus, 1925 * interrupt controller, debug module, PE reset or halt. 1926 */ 1927 else if (cs_etm__is_async_exception(tidq, magic)) 1928 packet->flags = PERF_IP_FLAG_BRANCH | 1929 PERF_IP_FLAG_CALL | 1930 PERF_IP_FLAG_ASYNC | 1931 PERF_IP_FLAG_INTERRUPT; 1932 /* 1933 * Otherwise, exception is caused by trap, instruction & 1934 * data fault, or alignment errors. 1935 */ 1936 else if (cs_etm__is_sync_exception(etmq, tidq, magic)) 1937 packet->flags = PERF_IP_FLAG_BRANCH | 1938 PERF_IP_FLAG_CALL | 1939 PERF_IP_FLAG_INTERRUPT; 1940 1941 /* 1942 * When the exception packet is inserted, since exception 1943 * packet is not used standalone for generating samples 1944 * and it's affiliation to the previous instruction range 1945 * packet; so set previous range packet flags to tell perf 1946 * it is an exception taken branch. 1947 */ 1948 if (prev_packet->sample_type == CS_ETM_RANGE) 1949 prev_packet->flags = packet->flags; 1950 break; 1951 case CS_ETM_EXCEPTION_RET: 1952 /* 1953 * When the exception return packet is inserted, since 1954 * exception return packet is not used standalone for 1955 * generating samples and it's affiliation to the previous 1956 * instruction range packet; so set previous range packet 1957 * flags to tell perf it is an exception return branch. 1958 * 1959 * The exception return can be for either system call or 1960 * other exception types; unfortunately the packet doesn't 1961 * contain exception type related info so we cannot decide 1962 * the exception type purely based on exception return packet. 1963 * If we record the exception number from exception packet and 1964 * reuse it for excpetion return packet, this is not reliable 1965 * due the trace can be discontinuity or the interrupt can 1966 * be nested, thus the recorded exception number cannot be 1967 * used for exception return packet for these two cases. 1968 * 1969 * For exception return packet, we only need to distinguish the 1970 * packet is for system call or for other types. Thus the 1971 * decision can be deferred when receive the next packet which 1972 * contains the return address, based on the return address we 1973 * can read out the previous instruction and check if it's a 1974 * system call instruction and then calibrate the sample flag 1975 * as needed. 1976 */ 1977 if (prev_packet->sample_type == CS_ETM_RANGE) 1978 prev_packet->flags = PERF_IP_FLAG_BRANCH | 1979 PERF_IP_FLAG_RETURN | 1980 PERF_IP_FLAG_INTERRUPT; 1981 break; 1982 case CS_ETM_EMPTY: 1983 default: 1984 break; 1985 } 1986 1987 return 0; 1988 } 1989 1990 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq) 1991 { 1992 int ret = 0; 1993 size_t processed = 0; 1994 1995 /* 1996 * Packets are decoded and added to the decoder's packet queue 1997 * until the decoder packet processing callback has requested that 1998 * processing stops or there is nothing left in the buffer. Normal 1999 * operations that stop processing are a timestamp packet or a full 2000 * decoder buffer queue. 2001 */ 2002 ret = cs_etm_decoder__process_data_block(etmq->decoder, 2003 etmq->offset, 2004 &etmq->buf[etmq->buf_used], 2005 etmq->buf_len, 2006 &processed); 2007 if (ret) 2008 goto out; 2009 2010 etmq->offset += processed; 2011 etmq->buf_used += processed; 2012 etmq->buf_len -= processed; 2013 2014 out: 2015 return ret; 2016 } 2017 2018 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, 2019 struct cs_etm_traceid_queue *tidq) 2020 { 2021 int ret; 2022 struct cs_etm_packet_queue *packet_queue; 2023 2024 packet_queue = &tidq->packet_queue; 2025 2026 /* Process each packet in this chunk */ 2027 while (1) { 2028 ret = cs_etm_decoder__get_packet(packet_queue, 2029 tidq->packet); 2030 if (ret <= 0) 2031 /* 2032 * Stop processing this chunk on 2033 * end of data or error 2034 */ 2035 break; 2036 2037 /* 2038 * Since packet addresses are swapped in packet 2039 * handling within below switch() statements, 2040 * thus setting sample flags must be called 2041 * prior to switch() statement to use address 2042 * information before packets swapping. 2043 */ 2044 ret = cs_etm__set_sample_flags(etmq, tidq); 2045 if (ret < 0) 2046 break; 2047 2048 switch (tidq->packet->sample_type) { 2049 case CS_ETM_RANGE: 2050 /* 2051 * If the packet contains an instruction 2052 * range, generate instruction sequence 2053 * events. 2054 */ 2055 cs_etm__sample(etmq, tidq); 2056 break; 2057 case CS_ETM_EXCEPTION: 2058 case CS_ETM_EXCEPTION_RET: 2059 /* 2060 * If the exception packet is coming, 2061 * make sure the previous instruction 2062 * range packet to be handled properly. 2063 */ 2064 cs_etm__exception(tidq); 2065 break; 2066 case CS_ETM_DISCONTINUITY: 2067 /* 2068 * Discontinuity in trace, flush 2069 * previous branch stack 2070 */ 2071 cs_etm__flush(etmq, tidq); 2072 break; 2073 case CS_ETM_EMPTY: 2074 /* 2075 * Should not receive empty packet, 2076 * report error. 2077 */ 2078 pr_err("CS ETM Trace: empty packet\n"); 2079 return -EINVAL; 2080 default: 2081 break; 2082 } 2083 } 2084 2085 return ret; 2086 } 2087 2088 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) 2089 { 2090 int idx; 2091 struct int_node *inode; 2092 struct cs_etm_traceid_queue *tidq; 2093 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 2094 2095 intlist__for_each_entry(inode, traceid_queues_list) { 2096 idx = (int)(intptr_t)inode->priv; 2097 tidq = etmq->traceid_queues[idx]; 2098 2099 /* Ignore return value */ 2100 cs_etm__process_traceid_queue(etmq, tidq); 2101 2102 /* 2103 * Generate an instruction sample with the remaining 2104 * branchstack entries. 2105 */ 2106 cs_etm__flush(etmq, tidq); 2107 } 2108 } 2109 2110 static int cs_etm__run_decoder(struct cs_etm_queue *etmq) 2111 { 2112 int err = 0; 2113 struct cs_etm_traceid_queue *tidq; 2114 2115 tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID); 2116 if (!tidq) 2117 return -EINVAL; 2118 2119 /* Go through each buffer in the queue and decode them one by one */ 2120 while (1) { 2121 err = cs_etm__get_data_block(etmq); 2122 if (err <= 0) 2123 return err; 2124 2125 /* Run trace decoder until buffer consumed or end of trace */ 2126 do { 2127 err = cs_etm__decode_data_block(etmq); 2128 if (err) 2129 return err; 2130 2131 /* 2132 * Process each packet in this chunk, nothing to do if 2133 * an error occurs other than hoping the next one will 2134 * be better. 2135 */ 2136 err = cs_etm__process_traceid_queue(etmq, tidq); 2137 2138 } while (etmq->buf_len); 2139 2140 if (err == 0) 2141 /* Flush any remaining branch stack entries */ 2142 err = cs_etm__end_block(etmq, tidq); 2143 } 2144 2145 return err; 2146 } 2147 2148 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, 2149 pid_t tid) 2150 { 2151 unsigned int i; 2152 struct auxtrace_queues *queues = &etm->queues; 2153 2154 for (i = 0; i < queues->nr_queues; i++) { 2155 struct auxtrace_queue *queue = &etm->queues.queue_array[i]; 2156 struct cs_etm_queue *etmq = queue->priv; 2157 struct cs_etm_traceid_queue *tidq; 2158 2159 if (!etmq) 2160 continue; 2161 2162 tidq = cs_etm__etmq_get_traceid_queue(etmq, 2163 CS_ETM_PER_THREAD_TRACEID); 2164 2165 if (!tidq) 2166 continue; 2167 2168 if ((tid == -1) || (tidq->tid == tid)) { 2169 cs_etm__set_pid_tid_cpu(etm, tidq); 2170 cs_etm__run_decoder(etmq); 2171 } 2172 } 2173 2174 return 0; 2175 } 2176 2177 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm) 2178 { 2179 int ret = 0; 2180 unsigned int cs_queue_nr, queue_nr; 2181 u8 trace_chan_id; 2182 u64 timestamp; 2183 struct auxtrace_queue *queue; 2184 struct cs_etm_queue *etmq; 2185 struct cs_etm_traceid_queue *tidq; 2186 2187 while (1) { 2188 if (!etm->heap.heap_cnt) 2189 goto out; 2190 2191 /* Take the entry at the top of the min heap */ 2192 cs_queue_nr = etm->heap.heap_array[0].queue_nr; 2193 queue_nr = TO_QUEUE_NR(cs_queue_nr); 2194 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr); 2195 queue = &etm->queues.queue_array[queue_nr]; 2196 etmq = queue->priv; 2197 2198 /* 2199 * Remove the top entry from the heap since we are about 2200 * to process it. 2201 */ 2202 auxtrace_heap__pop(&etm->heap); 2203 2204 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 2205 if (!tidq) { 2206 /* 2207 * No traceID queue has been allocated for this traceID, 2208 * which means something somewhere went very wrong. No 2209 * other choice than simply exit. 2210 */ 2211 ret = -EINVAL; 2212 goto out; 2213 } 2214 2215 /* 2216 * Packets associated with this timestamp are already in 2217 * the etmq's traceID queue, so process them. 2218 */ 2219 ret = cs_etm__process_traceid_queue(etmq, tidq); 2220 if (ret < 0) 2221 goto out; 2222 2223 /* 2224 * Packets for this timestamp have been processed, time to 2225 * move on to the next timestamp, fetching a new auxtrace_buffer 2226 * if need be. 2227 */ 2228 refetch: 2229 ret = cs_etm__get_data_block(etmq); 2230 if (ret < 0) 2231 goto out; 2232 2233 /* 2234 * No more auxtrace_buffers to process in this etmq, simply 2235 * move on to another entry in the auxtrace_heap. 2236 */ 2237 if (!ret) 2238 continue; 2239 2240 ret = cs_etm__decode_data_block(etmq); 2241 if (ret) 2242 goto out; 2243 2244 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); 2245 2246 if (!timestamp) { 2247 /* 2248 * Function cs_etm__decode_data_block() returns when 2249 * there is no more traces to decode in the current 2250 * auxtrace_buffer OR when a timestamp has been 2251 * encountered on any of the traceID queues. Since we 2252 * did not get a timestamp, there is no more traces to 2253 * process in this auxtrace_buffer. As such empty and 2254 * flush all traceID queues. 2255 */ 2256 cs_etm__clear_all_traceid_queues(etmq); 2257 2258 /* Fetch another auxtrace_buffer for this etmq */ 2259 goto refetch; 2260 } 2261 2262 /* 2263 * Add to the min heap the timestamp for packets that have 2264 * just been decoded. They will be processed and synthesized 2265 * during the next call to cs_etm__process_traceid_queue() for 2266 * this queue/traceID. 2267 */ 2268 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); 2269 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp); 2270 } 2271 2272 out: 2273 return ret; 2274 } 2275 2276 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm, 2277 union perf_event *event) 2278 { 2279 struct thread *th; 2280 2281 if (etm->timeless_decoding) 2282 return 0; 2283 2284 /* 2285 * Add the tid/pid to the log so that we can get a match when 2286 * we get a contextID from the decoder. 2287 */ 2288 th = machine__findnew_thread(etm->machine, 2289 event->itrace_start.pid, 2290 event->itrace_start.tid); 2291 if (!th) 2292 return -ENOMEM; 2293 2294 thread__put(th); 2295 2296 return 0; 2297 } 2298 2299 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm, 2300 union perf_event *event) 2301 { 2302 struct thread *th; 2303 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; 2304 2305 /* 2306 * Context switch in per-thread mode are irrelevant since perf 2307 * will start/stop tracing as the process is scheduled. 2308 */ 2309 if (etm->timeless_decoding) 2310 return 0; 2311 2312 /* 2313 * SWITCH_IN events carry the next process to be switched out while 2314 * SWITCH_OUT events carry the process to be switched in. As such 2315 * we don't care about IN events. 2316 */ 2317 if (!out) 2318 return 0; 2319 2320 /* 2321 * Add the tid/pid to the log so that we can get a match when 2322 * we get a contextID from the decoder. 2323 */ 2324 th = machine__findnew_thread(etm->machine, 2325 event->context_switch.next_prev_pid, 2326 event->context_switch.next_prev_tid); 2327 if (!th) 2328 return -ENOMEM; 2329 2330 thread__put(th); 2331 2332 return 0; 2333 } 2334 2335 static int cs_etm__process_event(struct perf_session *session, 2336 union perf_event *event, 2337 struct perf_sample *sample, 2338 struct perf_tool *tool) 2339 { 2340 int err = 0; 2341 u64 timestamp; 2342 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 2343 struct cs_etm_auxtrace, 2344 auxtrace); 2345 2346 if (dump_trace) 2347 return 0; 2348 2349 if (!tool->ordered_events) { 2350 pr_err("CoreSight ETM Trace requires ordered events\n"); 2351 return -EINVAL; 2352 } 2353 2354 if (sample->time && (sample->time != (u64) -1)) 2355 timestamp = sample->time; 2356 else 2357 timestamp = 0; 2358 2359 if (timestamp || etm->timeless_decoding) { 2360 err = cs_etm__update_queues(etm); 2361 if (err) 2362 return err; 2363 } 2364 2365 if (etm->timeless_decoding && 2366 event->header.type == PERF_RECORD_EXIT) 2367 return cs_etm__process_timeless_queues(etm, 2368 event->fork.tid); 2369 2370 if (event->header.type == PERF_RECORD_ITRACE_START) 2371 return cs_etm__process_itrace_start(etm, event); 2372 else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) 2373 return cs_etm__process_switch_cpu_wide(etm, event); 2374 2375 if (!etm->timeless_decoding && 2376 event->header.type == PERF_RECORD_AUX) 2377 return cs_etm__process_queues(etm); 2378 2379 return 0; 2380 } 2381 2382 static int cs_etm__process_auxtrace_event(struct perf_session *session, 2383 union perf_event *event, 2384 struct perf_tool *tool __maybe_unused) 2385 { 2386 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 2387 struct cs_etm_auxtrace, 2388 auxtrace); 2389 if (!etm->data_queued) { 2390 struct auxtrace_buffer *buffer; 2391 off_t data_offset; 2392 int fd = perf_data__fd(session->data); 2393 bool is_pipe = perf_data__is_pipe(session->data); 2394 int err; 2395 2396 if (is_pipe) 2397 data_offset = 0; 2398 else { 2399 data_offset = lseek(fd, 0, SEEK_CUR); 2400 if (data_offset == -1) 2401 return -errno; 2402 } 2403 2404 err = auxtrace_queues__add_event(&etm->queues, session, 2405 event, data_offset, &buffer); 2406 if (err) 2407 return err; 2408 2409 if (dump_trace) 2410 if (auxtrace_buffer__get_data(buffer, fd)) { 2411 cs_etm__dump_event(etm, buffer); 2412 auxtrace_buffer__put_data(buffer); 2413 } 2414 } 2415 2416 return 0; 2417 } 2418 2419 static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm) 2420 { 2421 struct evsel *evsel; 2422 struct evlist *evlist = etm->session->evlist; 2423 bool timeless_decoding = true; 2424 2425 /* 2426 * Circle through the list of event and complain if we find one 2427 * with the time bit set. 2428 */ 2429 evlist__for_each_entry(evlist, evsel) { 2430 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) 2431 timeless_decoding = false; 2432 } 2433 2434 return timeless_decoding; 2435 } 2436 2437 static const char * const cs_etm_global_header_fmts[] = { 2438 [CS_HEADER_VERSION_0] = " Header version %llx\n", 2439 [CS_PMU_TYPE_CPUS] = " PMU type/num cpus %llx\n", 2440 [CS_ETM_SNAPSHOT] = " Snapshot %llx\n", 2441 }; 2442 2443 static const char * const cs_etm_priv_fmts[] = { 2444 [CS_ETM_MAGIC] = " Magic number %llx\n", 2445 [CS_ETM_CPU] = " CPU %lld\n", 2446 [CS_ETM_ETMCR] = " ETMCR %llx\n", 2447 [CS_ETM_ETMTRACEIDR] = " ETMTRACEIDR %llx\n", 2448 [CS_ETM_ETMCCER] = " ETMCCER %llx\n", 2449 [CS_ETM_ETMIDR] = " ETMIDR %llx\n", 2450 }; 2451 2452 static const char * const cs_etmv4_priv_fmts[] = { 2453 [CS_ETM_MAGIC] = " Magic number %llx\n", 2454 [CS_ETM_CPU] = " CPU %lld\n", 2455 [CS_ETMV4_TRCCONFIGR] = " TRCCONFIGR %llx\n", 2456 [CS_ETMV4_TRCTRACEIDR] = " TRCTRACEIDR %llx\n", 2457 [CS_ETMV4_TRCIDR0] = " TRCIDR0 %llx\n", 2458 [CS_ETMV4_TRCIDR1] = " TRCIDR1 %llx\n", 2459 [CS_ETMV4_TRCIDR2] = " TRCIDR2 %llx\n", 2460 [CS_ETMV4_TRCIDR8] = " TRCIDR8 %llx\n", 2461 [CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n", 2462 }; 2463 2464 static void cs_etm__print_auxtrace_info(__u64 *val, int num) 2465 { 2466 int i, j, cpu = 0; 2467 2468 for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++) 2469 fprintf(stdout, cs_etm_global_header_fmts[i], val[i]); 2470 2471 for (i = CS_HEADER_VERSION_0_MAX; cpu < num; cpu++) { 2472 if (val[i] == __perf_cs_etmv3_magic) 2473 for (j = 0; j < CS_ETM_PRIV_MAX; j++, i++) 2474 fprintf(stdout, cs_etm_priv_fmts[j], val[i]); 2475 else if (val[i] == __perf_cs_etmv4_magic) 2476 for (j = 0; j < CS_ETMV4_PRIV_MAX; j++, i++) 2477 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]); 2478 else 2479 /* failure.. return */ 2480 return; 2481 } 2482 } 2483 2484 int cs_etm__process_auxtrace_info(union perf_event *event, 2485 struct perf_session *session) 2486 { 2487 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; 2488 struct cs_etm_auxtrace *etm = NULL; 2489 struct int_node *inode; 2490 unsigned int pmu_type; 2491 int event_header_size = sizeof(struct perf_event_header); 2492 int info_header_size; 2493 int total_size = auxtrace_info->header.size; 2494 int priv_size = 0; 2495 int num_cpu; 2496 int err = 0, idx = -1; 2497 int i, j, k; 2498 u64 *ptr, *hdr = NULL; 2499 u64 **metadata = NULL; 2500 2501 /* 2502 * sizeof(auxtrace_info_event::type) + 2503 * sizeof(auxtrace_info_event::reserved) == 8 2504 */ 2505 info_header_size = 8; 2506 2507 if (total_size < (event_header_size + info_header_size)) 2508 return -EINVAL; 2509 2510 priv_size = total_size - event_header_size - info_header_size; 2511 2512 /* First the global part */ 2513 ptr = (u64 *) auxtrace_info->priv; 2514 2515 /* Look for version '0' of the header */ 2516 if (ptr[0] != 0) 2517 return -EINVAL; 2518 2519 hdr = zalloc(sizeof(*hdr) * CS_HEADER_VERSION_0_MAX); 2520 if (!hdr) 2521 return -ENOMEM; 2522 2523 /* Extract header information - see cs-etm.h for format */ 2524 for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++) 2525 hdr[i] = ptr[i]; 2526 num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff; 2527 pmu_type = (unsigned int) ((hdr[CS_PMU_TYPE_CPUS] >> 32) & 2528 0xffffffff); 2529 2530 /* 2531 * Create an RB tree for traceID-metadata tuple. Since the conversion 2532 * has to be made for each packet that gets decoded, optimizing access 2533 * in anything other than a sequential array is worth doing. 2534 */ 2535 traceid_list = intlist__new(NULL); 2536 if (!traceid_list) { 2537 err = -ENOMEM; 2538 goto err_free_hdr; 2539 } 2540 2541 metadata = zalloc(sizeof(*metadata) * num_cpu); 2542 if (!metadata) { 2543 err = -ENOMEM; 2544 goto err_free_traceid_list; 2545 } 2546 2547 /* 2548 * The metadata is stored in the auxtrace_info section and encodes 2549 * the configuration of the ARM embedded trace macrocell which is 2550 * required by the trace decoder to properly decode the trace due 2551 * to its highly compressed nature. 2552 */ 2553 for (j = 0; j < num_cpu; j++) { 2554 if (ptr[i] == __perf_cs_etmv3_magic) { 2555 metadata[j] = zalloc(sizeof(*metadata[j]) * 2556 CS_ETM_PRIV_MAX); 2557 if (!metadata[j]) { 2558 err = -ENOMEM; 2559 goto err_free_metadata; 2560 } 2561 for (k = 0; k < CS_ETM_PRIV_MAX; k++) 2562 metadata[j][k] = ptr[i + k]; 2563 2564 /* The traceID is our handle */ 2565 idx = metadata[j][CS_ETM_ETMTRACEIDR]; 2566 i += CS_ETM_PRIV_MAX; 2567 } else if (ptr[i] == __perf_cs_etmv4_magic) { 2568 metadata[j] = zalloc(sizeof(*metadata[j]) * 2569 CS_ETMV4_PRIV_MAX); 2570 if (!metadata[j]) { 2571 err = -ENOMEM; 2572 goto err_free_metadata; 2573 } 2574 for (k = 0; k < CS_ETMV4_PRIV_MAX; k++) 2575 metadata[j][k] = ptr[i + k]; 2576 2577 /* The traceID is our handle */ 2578 idx = metadata[j][CS_ETMV4_TRCTRACEIDR]; 2579 i += CS_ETMV4_PRIV_MAX; 2580 } 2581 2582 /* Get an RB node for this CPU */ 2583 inode = intlist__findnew(traceid_list, idx); 2584 2585 /* Something went wrong, no need to continue */ 2586 if (!inode) { 2587 err = -ENOMEM; 2588 goto err_free_metadata; 2589 } 2590 2591 /* 2592 * The node for that CPU should not be taken. 2593 * Back out if that's the case. 2594 */ 2595 if (inode->priv) { 2596 err = -EINVAL; 2597 goto err_free_metadata; 2598 } 2599 /* All good, associate the traceID with the metadata pointer */ 2600 inode->priv = metadata[j]; 2601 } 2602 2603 /* 2604 * Each of CS_HEADER_VERSION_0_MAX, CS_ETM_PRIV_MAX and 2605 * CS_ETMV4_PRIV_MAX mark how many double words are in the 2606 * global metadata, and each cpu's metadata respectively. 2607 * The following tests if the correct number of double words was 2608 * present in the auxtrace info section. 2609 */ 2610 if (i * 8 != priv_size) { 2611 err = -EINVAL; 2612 goto err_free_metadata; 2613 } 2614 2615 etm = zalloc(sizeof(*etm)); 2616 2617 if (!etm) { 2618 err = -ENOMEM; 2619 goto err_free_metadata; 2620 } 2621 2622 err = auxtrace_queues__init(&etm->queues); 2623 if (err) 2624 goto err_free_etm; 2625 2626 etm->session = session; 2627 etm->machine = &session->machines.host; 2628 2629 etm->num_cpu = num_cpu; 2630 etm->pmu_type = pmu_type; 2631 etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0); 2632 etm->metadata = metadata; 2633 etm->auxtrace_type = auxtrace_info->type; 2634 etm->timeless_decoding = cs_etm__is_timeless_decoding(etm); 2635 2636 etm->auxtrace.process_event = cs_etm__process_event; 2637 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event; 2638 etm->auxtrace.flush_events = cs_etm__flush_events; 2639 etm->auxtrace.free_events = cs_etm__free_events; 2640 etm->auxtrace.free = cs_etm__free; 2641 etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace; 2642 session->auxtrace = &etm->auxtrace; 2643 2644 etm->unknown_thread = thread__new(999999999, 999999999); 2645 if (!etm->unknown_thread) { 2646 err = -ENOMEM; 2647 goto err_free_queues; 2648 } 2649 2650 /* 2651 * Initialize list node so that at thread__zput() we can avoid 2652 * segmentation fault at list_del_init(). 2653 */ 2654 INIT_LIST_HEAD(&etm->unknown_thread->node); 2655 2656 err = thread__set_comm(etm->unknown_thread, "unknown", 0); 2657 if (err) 2658 goto err_delete_thread; 2659 2660 if (thread__init_maps(etm->unknown_thread, etm->machine)) { 2661 err = -ENOMEM; 2662 goto err_delete_thread; 2663 } 2664 2665 if (dump_trace) { 2666 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu); 2667 return 0; 2668 } 2669 2670 if (session->itrace_synth_opts->set) { 2671 etm->synth_opts = *session->itrace_synth_opts; 2672 } else { 2673 itrace_synth_opts__set_default(&etm->synth_opts, 2674 session->itrace_synth_opts->default_no_sample); 2675 etm->synth_opts.callchain = false; 2676 } 2677 2678 err = cs_etm__synth_events(etm, session); 2679 if (err) 2680 goto err_delete_thread; 2681 2682 err = auxtrace_queues__process_index(&etm->queues, session); 2683 if (err) 2684 goto err_delete_thread; 2685 2686 etm->data_queued = etm->queues.populated; 2687 2688 return 0; 2689 2690 err_delete_thread: 2691 thread__zput(etm->unknown_thread); 2692 err_free_queues: 2693 auxtrace_queues__free(&etm->queues); 2694 session->auxtrace = NULL; 2695 err_free_etm: 2696 zfree(&etm); 2697 err_free_metadata: 2698 /* No need to check @metadata[j], free(NULL) is supported */ 2699 for (j = 0; j < num_cpu; j++) 2700 zfree(&metadata[j]); 2701 zfree(&metadata); 2702 err_free_traceid_list: 2703 intlist__delete(traceid_list); 2704 err_free_hdr: 2705 zfree(&hdr); 2706 2707 return err; 2708 } 2709